From 3552afd96cc8d90180fd56f842bd4efff9420dda Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Mon, 24 Aug 2020 11:09:47 +0900 Subject: [PATCH] [init] Update weight initializations Add proper bias initialization than just bias_init_zero Update weight_ini to weight_initializer Add zero initializer to the list of supported initiailizers **Self evaluation:** 1. Build test: [x]Passed [ ]Failed [ ]Skipped 2. Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Parichay Kapoor --- Applications/Classification/README.md | 4 +- Applications/Classification/res/Classification.ini | 4 +- .../Classification/res/Classification_func.ini | 4 +- .../Classification/res/Classification_new.ini | 6 +-- .../LogisticRegression/res/LogisticRegression.ini | 4 +- .../ReinforcementLearning/DeepQ/jni/DeepQ.ini | 8 +-- Applications/Tizen_CAPI/Tizen_CAPI_config.ini | 4 +- Applications/Tizen_CAPI/capi_file.c | 6 +-- Applications/Tizen_CAPI/capi_func.c | 6 +-- Applications/Training/res/Training.ini | 6 +-- Applications/mnist/res/mnist.ini | 12 ++--- Applications/mnist/res/mnist_valid.ini | 12 ++--- README.md | 10 ++-- api/capi/include/nntrainer.h | 4 +- docs/configuration-ini.md | 33 ++++++------ nntrainer/include/layer.h | 63 ++++++++++------------ nntrainer/include/neuralnet.h | 4 +- nntrainer/include/parse_util.h | 4 +- nntrainer/src/conv2d_layer.cpp | 10 ++-- nntrainer/src/fc_layer.cpp | 13 +++-- nntrainer/src/layer.cpp | 28 +++++----- nntrainer/src/neuralnet.cpp | 2 +- nntrainer/src/parse_util.cpp | 60 +++++++++++++-------- test/include/nntrainer_test_util.h | 10 ++-- test/tizen_capi/test_conf.ini | 6 +-- test/tizen_capi/unittest_tizen_capi.cpp | 34 ++++++------ test/tizen_capi/unittest_tizen_capi_layer.cpp | 2 +- test/unittest/unittest_nntrainer_internal.cpp | 4 +- test/unittest/unittest_nntrainer_layers.cpp | 22 ++++---- test/unittest/unittest_nntrainer_modelfile.cpp | 6 +-- 30 files changed, 199 insertions(+), 192 deletions(-) diff --git a/Applications/Classification/README.md b/Applications/Classification/README.md index b8470bc..44b0e1e 100644 --- a/Applications/Classification/README.md +++ b/Applications/Classification/README.md @@ -69,13 +69,13 @@ LabelData="label.dat" [inputlayer] Type = input Input_Shape = 1:1:62720 -Bias_init_zero = true +bias_initializer = true Normalization = true [outputlayer] Type = fully_connected Unit = 10 -Bias_init_zero = true +bias_initializer = true Activation = softmax Weight_Decay = l2norm weight_Decay_Lambda = 0.005 diff --git a/Applications/Classification/res/Classification.ini b/Applications/Classification/res/Classification.ini index 48bd139..466de50 100644 --- a/Applications/Classification/res/Classification.ini +++ b/Applications/Classification/res/Classification.ini @@ -24,13 +24,13 @@ LabelData="label.dat" [inputlayer] Type = input Input_Shape = 1:1:62720 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Normalization = true [outputlayer] Type = fully_connected Unit = 10 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = softmax # activation : sigmoid, softmax Weight_Decay = l2norm weight_Decay_Lambda = 0.005 diff --git a/Applications/Classification/res/Classification_func.ini b/Applications/Classification/res/Classification_func.ini index 9fc201d..4072cef 100644 --- a/Applications/Classification/res/Classification_func.ini +++ b/Applications/Classification/res/Classification_func.ini @@ -19,14 +19,14 @@ epsilon = 1e-7 # epsilon for adam [inputlayer] Type = input Input_Shape = 1:1:62720 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Normalization = true Activation = sigmoid # activation : sigmoid, tanh [outputlayer] Type = fully_connected unit = 10 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = softmax # activation : sigmoid, softmax Weight_Decay = l2norm weight_Decay_Lambda = 0.005 diff --git a/Applications/Classification/res/Classification_new.ini b/Applications/Classification/res/Classification_new.ini index c2234fc..c735ce0 100644 --- a/Applications/Classification/res/Classification_new.ini +++ b/Applications/Classification/res/Classification_new.ini @@ -20,12 +20,12 @@ epsilon = 1e-8 # epsilon for adam [inputlayer] Type = InputLayer HiddenSize = 62720 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias [fc1layer] Type = FullyConnectedLayer HiddenSize = 128 # Hidden Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros [batchnormalization] Type = BatchNormalizationLayer @@ -33,5 +33,5 @@ Type = BatchNormalizationLayer [outputlayer] Type = OutputLayer HiddenSize = 10 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Softmax = true diff --git a/Applications/LogisticRegression/res/LogisticRegression.ini b/Applications/LogisticRegression/res/LogisticRegression.ini index 2f227f6..9c31183 100644 --- a/Applications/LogisticRegression/res/LogisticRegression.ini +++ b/Applications/LogisticRegression/res/LogisticRegression.ini @@ -15,11 +15,11 @@ epsilon = 1e-5 [inputlayer] Type = input Input_Shape = 1:1:2 -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Activation = sigmoid [outputlayer] Type = fully_connected Unit = 1 -Bias_init_zero = true +Bias_initializer = zeros Activation = sigmoid diff --git a/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini b/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini index 1b249e4..e984880 100644 --- a/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini +++ b/Applications/ReinforcementLearning/DeepQ/jni/DeepQ.ini @@ -18,23 +18,23 @@ epsilon = 1e-8 # epsilon for adam [inputlayer] Type = input Input_Shape = 1:1:4 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Activation = tanh # activation : sigmoid, tanh [fc1layer] Type = fully_connected Unit = 50 # Hidden Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = tanh # activation : sigmoid, tanh [fc2layer] Type = fully_connected Unit = 50 -Bias_init_zero = true +Bias_initializer = zeros Activation = tanh # activation : sigmoid, tanh [outputlayer] Type = fully_connected Unit = 2 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = sigmoid # activation : sigmoid, tanh diff --git a/Applications/Tizen_CAPI/Tizen_CAPI_config.ini b/Applications/Tizen_CAPI/Tizen_CAPI_config.ini index 072e421..e7e865b 100644 --- a/Applications/Tizen_CAPI/Tizen_CAPI_config.ini +++ b/Applications/Tizen_CAPI/Tizen_CAPI_config.ini @@ -30,14 +30,14 @@ LabelData="label.dat" [inputlayer] Type = input Input_Shape = 1:1:62720 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Normalization = true Activation = sigmoid # activation : sigmoid, tanh [outputlayer] Type = fully_connected Unit = 10 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = softmax # activation : sigmoid, softmax Weight_Decay = l2norm weight_Decay_Lambda = 0.005 diff --git a/Applications/Tizen_CAPI/capi_file.c b/Applications/Tizen_CAPI/capi_file.c index 72766c3..e9686ed 100644 --- a/Applications/Tizen_CAPI/capi_file.c +++ b/Applications/Tizen_CAPI/capi_file.c @@ -54,7 +54,7 @@ int main(int argc, char *argv[]) { /* set property for input layer */ status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); NN_RETURN_STATUS(); /* add input layer into model */ @@ -67,9 +67,9 @@ int main(int argc, char *argv[]) { /* set property for fc layer */ status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", - "weight_ini=xavier_uniform", NULL); + "weight_initializer=xavier_uniform", NULL); NN_RETURN_STATUS(); /* add fc layer into model */ diff --git a/Applications/Tizen_CAPI/capi_func.c b/Applications/Tizen_CAPI/capi_func.c index 388b8f4..7e4bcbf 100644 --- a/Applications/Tizen_CAPI/capi_func.c +++ b/Applications/Tizen_CAPI/capi_func.c @@ -307,7 +307,7 @@ int main(int argc, char *argv[]) { /* set property for input layer */ status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); NN_RETURN_STATUS(); /* add input layer into model */ @@ -320,9 +320,9 @@ int main(int argc, char *argv[]) { /* set property for fc layer */ status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", - "weight_ini=xavier_uniform", NULL); + "weight_initializer=xavier_uniform", NULL); NN_RETURN_STATUS(); /* add fc layer into model */ diff --git a/Applications/Training/res/Training.ini b/Applications/Training/res/Training.ini index 52eac60..8cbbd81 100644 --- a/Applications/Training/res/Training.ini +++ b/Applications/Training/res/Training.ini @@ -17,17 +17,17 @@ minibatch = 1 # mini batch size [inputlayer] Type = input Input_Shape = 1:1:128 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Activation = sigmoid [fc1layer] Type = fully_connected Unit = 20 # Hidden Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = sigmoid [outputlayer] Type = fully_connected Unit = 3 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = sigmoid diff --git a/Applications/mnist/res/mnist.ini b/Applications/mnist/res/mnist.ini index cb10de1..6209b15 100644 --- a/Applications/mnist/res/mnist.ini +++ b/Applications/mnist/res/mnist.ini @@ -22,9 +22,9 @@ Input_Shape = 1:28:28 [conv2d_c1_layer] Type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer=zeros Activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 6 stride = 1,1 padding = 0,0 @@ -39,9 +39,9 @@ pooling = average [conv2d_c2_layer] Type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer=zeros Activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 12 stride = 1,1 padding = 0,0 @@ -59,6 +59,6 @@ Type=flatten [outputlayer] Type = fully_connected Unit = 10 # Output Layer Dimension ( = Weight Width ) -weight_ini = xavier_uniform -bias_init_zero = true +weight_initializer = xavier_uniform +bias_initializer = zeros Activation = softmax # activation : sigmoid, softmax diff --git a/Applications/mnist/res/mnist_valid.ini b/Applications/mnist/res/mnist_valid.ini index 958f5a8..fb73c3c 100644 --- a/Applications/mnist/res/mnist_valid.ini +++ b/Applications/mnist/res/mnist_valid.ini @@ -22,9 +22,9 @@ Input_Shape = 1:28:28 [conv2d_c1_layer] Type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer = zeros Activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 6 stride = 1,1 padding = 0,0 @@ -39,9 +39,9 @@ pooling = average [conv2d_c2_layer] Type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer = zeros Activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 12 stride = 1,1 padding = 0,0 @@ -59,6 +59,6 @@ Type=flatten [outputlayer] Type = fully_connected Unit = 10 # Output Layer Dimension ( = Weight Width ) -weight_ini = xavier_uniform -bias_init_zero = true +weight_initializer = xavier_uniform +bias_initializer = zeros Activation = softmax # activation : sigmoid, softmax diff --git a/README.md b/README.md index 1cdc45b..6353ab4 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ This component defines Layers which consist of Neural Network Model. Layers has | input | Input | Input Layer. This is not always requied. | | batch_normalization | Batch Normalization Layer | Batch Normalization Layer. | | loss layer | loss layer | hidden from users | - | activation | activaiton layer | set by layer property | + | activation | activaiton layer | set by layer property | ### Supported Optimizers @@ -75,9 +75,9 @@ NNTrainer provides | sigmoid | sigmoid function | set as layer property | | relu | relu function | set as layer propery | | softmax | softmax function | set as layer propery | - | weight_ini | Weight Initialization | Xavier(Normal/Uniform), LeCun(Normal/Uniform), HE(Normal/Unifor) | + | weight_initializer | Weight Initialization | Xavier(Normal/Uniform), LeCun(Normal/Uniform), HE(Normal/Unifor) | | weight_decay | weight decay ( L2Norm only ) | needs set weight_decay_param & type | - | learnig_rate_decay | learning rate decay | need to set step | + | learnig_rate_decay | learning rate decay | need to set step | ### Tensor @@ -99,9 +99,9 @@ NNTrainer provides | Keyward | Loss Name | Description | |:-------:|:---:|:---| - | weight_ini | Weight Initialization | Xavier(Normal/Uniform), LeCun(Normal/Uniform), HE(Normal/Unifor) | + | weight_initializer | Weight Initialization | Xavier(Normal/Uniform), LeCun(Normal/Uniform), HE(Normal/Unifor) | | weight_decay | weight decay ( L2Norm only ) | needs set weight_decay_param & type | - | learnig_rate_decay | learning rate decay | need to set step | + | learnig_rate_decay | learning rate decay | need to set step | ### APIs Currently we privde [C APIs](https://github.com/nnstreamer/nntrainer/blob/master/api/capi/include/nntrainer.h) for Tizen. C++ API also provides soon. diff --git a/api/capi/include/nntrainer.h b/api/capi/include/nntrainer.h index 0e65374..eca2ae8 100644 --- a/api/capi/include/nntrainer.h +++ b/api/capi/include/nntrainer.h @@ -288,8 +288,8 @@ int ml_train_layer_destroy(ml_train_layer_h layer); * * // Many of these hyperparmeters are optional * status = ml_train_layer_set_property(handle, "input_shape=1:1:6270", - * "unit=10", "bias_init_zero=true", "activation=sigmoid", - * "weight_decay=l2_norm", "weight_ini=he_uniform", NULL); + * "unit=10", "bias_initializer=zeros", "activation=sigmoid", + * "weight_decay=l2_norm", "weight_initializer=he_uniform", NULL); * if (status != ML_ERROR_NONE) { * // Handle error case * ml_train_layer_destroy(handle); diff --git a/docs/configuration-ini.md b/docs/configuration-ini.md index af35be1..bb72f7a 100644 --- a/docs/configuration-ini.md +++ b/docs/configuration-ini.md @@ -176,9 +176,10 @@ Start with "[ ${layer name} ]". This layer name must be unique throughout networ set the output layer for fully connected layer -11. ```weight_ini = ``` +11. ```weight_initializer = ``` set weight initialization method + * zeros : Zero initialization * lecun_normal : LeCun normal initialization * lecun_uniform : LeCun uniform initialization * xavier_normal : xavier normal initialization @@ -226,12 +227,12 @@ Each layer requires different properties. | Layer | Properties | |:-------:|:---| - | conv2d |filter
kernel_size
stride
padding
normalization
standardization
input_shape
bias_init_zero
activation
flatten
weight_decay
weight_decay_lambda
weight_ini
| - | pooling2d | pooling
pooling_size
stride
padding
 | + | conv2d |
  • filter
  • kernel_size
  • stride
  • padding
  • normalization
  • standardization
  • input_shape
  • bias_initializer
  • activation
  • flatten
  • weight_decay
  • weight_decay_lambda
  • weight_initializer
| + | pooling2d |
  • pooling
  • pooling_size
  • stride
  • padding
| | flatten | - | - | fully_connected | unit
normalization
standardization
input_shape
bias_init_zero
activation
flatten
weight_decay
weight_decay_lambda
weight_ini
| - | input | normalization
standardization
input_shape
flatten
| - | batch_normalization | epsilon
flatten
 | + | fully_connected |
  • unit
  • normalization
  • standardization
  • input_shape
  • bias_initializer
  • activation
  • flatten
  • weight_decay
  • weight_decay_lambda
  • weight_initializer
    • | + | input |
  • normalization
  • standardization
  • input_shape
  • flatten
    • | + | batch_normalization |
  • epsilon
  • flatten
    •  | Below is sample for layers to define a model. @@ -239,9 +240,9 @@ Below is sample for layers to define a model. [conv2d_c2_layer] Type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer=zeros Activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 12 stride = 1,1 padding = 0,0 @@ -249,8 +250,8 @@ padding = 0,0 [outputlayer] Type = fully_connected Unit = 10 -weight_ini = xavier_uniform -bias_init_zero = true +weight_initializer = xavier_uniform +bias_initializer = zeros Activation = softmax ``` @@ -282,9 +283,9 @@ input_shape = 1:28:28 [conv2d_c1_layer] type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer=zeros activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 6 stride = 1,1 padding = 0,0 @@ -299,9 +300,9 @@ pooling = average [conv2d_c2_layer] type = conv2d kernel_size = 5,5 -bias_init_zero=true +bias_initializer=zeros activation=sigmoid -weight_ini = xavier_uniform +weight_initializer = xavier_uniform filter = 12 stride = 1,1 padding = 0,0 @@ -319,7 +320,7 @@ type=flatten [outputlayer] type = fully_connected unit = 10 -weight_ini = xavier_uniform -bias_init_zero = true +weight_initializer = xavier_uniform +bias_initializer = zeros activation = softmax ``` diff --git a/nntrainer/include/layer.h b/nntrainer/include/layer.h index 149cfa5..fad4a4d 100644 --- a/nntrainer/include/layer.h +++ b/nntrainer/include/layer.h @@ -96,14 +96,16 @@ typedef enum { /** * @brief Enumeration of Weight Initialization Type - * 0. WEIGHT_LECUN_NORMAL ( LeCun normal initialization ) - * 1. WEIGHT_LECUN_UNIFORM (LeCun uniform initialization ) - * 2. WEIGHT_XAVIER_NORMAL ( Xavier normal initialization ) - * 3. WEIGHT_XAVIER_UNIFORM ( Xavier uniform initialization ) - * 4. WEIGHT_HE_NORMAL ( He normal initialization ) - * 5. WEIGHT_HE_UNIFORM ( He uniform initialization ) + * 0. WEIGHT_ZEROS ( Zero initialization ) + * 1. WEIGHT_LECUN_NORMAL ( LeCun normal initialization ) + * 2. WEIGHT_LECUN_UNIFORM (LeCun uniform initialization ) + * 3. WEIGHT_XAVIER_NORMAL ( Xavier normal initialization ) + * 4. WEIGHT_XAVIER_UNIFORM ( Xavier uniform initialization ) + * 5. WEIGHT_HE_NORMAL ( He normal initialization ) + * 6. WEIGHT_HE_UNIFORM ( He uniform initialization ) */ typedef enum { + WEIGHT_ZEROS, WEIGHT_LECUN_NORMAL, WEIGHT_LECUN_UNIFORM, WEIGHT_XAVIER_NORMAL, @@ -111,7 +113,7 @@ typedef enum { WEIGHT_HE_NORMAL, WEIGHT_HE_UNIFORM, WEIGHT_UNKNOWN -} WeightIniType; +} WeightInitializer; /** * @brief Print Options when printing layer info @@ -136,12 +138,12 @@ class Layer { public: Layer() : name(std::string()), - bias_init_zero(false), type(LAYER_UNKNOWN), loss(0.0f), activation_type(ACT_NONE), weight_decay(), - weight_ini_type(WEIGHT_XAVIER_UNIFORM), + weight_initializer(WEIGHT_XAVIER_UNIFORM), + bias_initializer(WEIGHT_ZEROS), flatten(false), trainable(true), param_size(0), @@ -222,7 +224,7 @@ public: * 6. weight_decay : string (type) * 7. weight_decay_lambda : float * 8. unit : int - * 9. weight_ini : string (type) + * 9. weight_initializer : string (type) * 10. filter_size : int * 11. kernel_size : ( n , m ) * 12. stride : ( n, m ) @@ -237,15 +239,15 @@ public: */ enum class PropertyType { input_shape = 0, - bias_init_zero = 1, - normalization = 2, - standardization = 3, - activation = 4, - epsilon = 5, - weight_decay = 6, - weight_decay_lambda = 7, - unit = 8, - weight_ini = 9, + normalization = 1, + standardization = 2, + activation = 3, + epsilon = 4, + weight_decay = 5, + weight_decay_lambda = 6, + unit = 7, + weight_initializer = 8, + bias_initializer = 9, filter = 10, kernel_size = 11, stride = 12, @@ -327,25 +329,19 @@ public: void setWeightDecay(WeightDecayParam w) { weight_decay = w; } /** - * @brief set bias initialize with zero - * @param[in] zero true/false - */ - void setBiasZero(bool zero) { bias_init_zero = zero; } - - /** * @brief set Weight Initialization Type - * @param[in] wini WeightIniType + * @param[in] wini WeightInitializer */ - void setWeightInit(WeightIniType wini) { weight_ini_type = wini; } + void setWeightInit(WeightInitializer wini) { weight_initializer = wini; } /** * @brief initialize Weight * @param[in] w_dim TensorDim - * @param[in] init_type Weight Initialization Type + * @param[in] initializer Weight Initializer * @param[out] status Status * @retval Tensor Initialized Tensor */ - Tensor initializeWeight(TensorDim w_dim, WeightIniType init_type, + Tensor initializeWeight(TensorDim w_dim, WeightInitializer initializer, int &status); /** @@ -483,11 +479,6 @@ protected: Optimizer opt; /** - * @brief Boolean for the Bias to set zero - */ - bool bias_init_zero; - - /** * @brief Layer type */ LayerType type; @@ -501,7 +492,9 @@ protected: WeightDecayParam weight_decay; - WeightIniType weight_ini_type; + WeightInitializer weight_initializer; /** initializer for weights */ + + WeightInitializer bias_initializer; /** initializer for bias */ /** * @brief Output of this layer should be flattened diff --git a/nntrainer/include/neuralnet.h b/nntrainer/include/neuralnet.h index 083c5f5..144fb2f 100644 --- a/nntrainer/include/neuralnet.h +++ b/nntrainer/include/neuralnet.h @@ -77,7 +77,7 @@ public: swap(lhs.epoch, rhs.epoch); swap(lhs.loss, rhs.loss); swap(lhs.cost, rhs.cost); - swap(lhs.weight_ini, rhs.weight_ini); + swap(lhs.weight_initializer, rhs.weight_initializer); swap(lhs.save_path, rhs.save_path); swap(lhs.opt, rhs.opt); swap(lhs.net_type, rhs.net_type); @@ -304,7 +304,7 @@ private: CostType cost; /**< Cost Function type */ - WeightIniType weight_ini; /**< Weight Initialization type */ + WeightInitializer weight_initializer; /**< Weight Initialization type */ std::string save_path; /**< Model path to save / read */ diff --git a/nntrainer/include/parse_util.h b/nntrainer/include/parse_util.h index 82beb37..b1377fa 100644 --- a/nntrainer/include/parse_util.h +++ b/nntrainer/include/parse_util.h @@ -44,7 +44,7 @@ namespace nntrainer { * 2. MODEL ( Model Token ) * 3. ACTI ( Activation Token ) * 4. LAYER ( Layer Token ) - * 5. WEIGHTINI ( Weight Initialization Token ) + * 5. WEIGHT_INIT ( Weight Initializer Token ) * 7. WEIGHT_DECAY ( Weight Decay Token ) * 8. PADDING ( Padding Token ) * 9. POOLING ( Pooling Token ) @@ -56,7 +56,7 @@ typedef enum { TOKEN_MODEL, TOKEN_ACTI, TOKEN_LAYER, - TOKEN_WEIGHTINI, + TOKEN_WEIGHT_INIT, TOKEN_WEIGHT_DECAY, TOKEN_PADDING, TOKEN_POOLING, diff --git a/nntrainer/src/conv2d_layer.cpp b/nntrainer/src/conv2d_layer.cpp index edf2841..5591f64 100644 --- a/nntrainer/src/conv2d_layer.cpp +++ b/nntrainer/src/conv2d_layer.cpp @@ -38,16 +38,12 @@ int Conv2DLayer::initialize() { setParamSize(filter_size * 2); for (unsigned int i = 0; i < filter_size; ++i) { - Tensor Knl = initializeWeight(dim, weight_ini_type, status); + Tensor Knl = initializeWeight(dim, weight_initializer, status); NN_RETURN_STATUS(); - Tensor bias = Tensor(bias_dim); + Tensor bias = initializeWeight(bias_dim, bias_initializer, status); + NN_RETURN_STATUS(); - if (!bias_init_zero) { - bias.apply([&](float x) { return random(); }); - } else { - bias.setZero(); - } Tensor delK(dim); delK.setZero(); diff --git a/nntrainer/src/fc_layer.cpp b/nntrainer/src/fc_layer.cpp index b95a094..07cda2d 100644 --- a/nntrainer/src/fc_layer.cpp +++ b/nntrainer/src/fc_layer.cpp @@ -39,18 +39,17 @@ int FullyConnectedLayer::initialize() { output_dim = input_dim; output_dim.width(unit); - Tensor bias = Tensor(1, unit); + TensorDim bias_dim = TensorDim(); + bias_dim.setTensorDim(3, unit); + TensorDim dim = output_dim; dim.height(input_dim.width()); dim.batch(1); - Tensor weight = initializeWeight(dim, weight_ini_type, status); + Tensor weight = initializeWeight(dim, weight_initializer, status); NN_RETURN_STATUS(); - if (bias_init_zero) { - bias.setZero(); - } else { - bias.setRandUniform(-0.5, 0.5); - } + Tensor bias = initializeWeight(bias_dim, bias_initializer, status); + NN_RETURN_STATUS(); setParamSize(2); paramsAt(0) = {std::move(weight), Tensor(weight.getDim()), "FC:weight"}; diff --git a/nntrainer/src/layer.cpp b/nntrainer/src/layer.cpp index a899acb..12cbc5c 100644 --- a/nntrainer/src/layer.cpp +++ b/nntrainer/src/layer.cpp @@ -81,18 +81,21 @@ void Layer::save(std::ofstream &file) { } } -Tensor Layer::initializeWeight(TensorDim w_dim, WeightIniType init_type, +Tensor Layer::initializeWeight(TensorDim w_dim, WeightInitializer initializer, int &status) { Tensor w = Tensor(w_dim); - if (init_type == WEIGHT_UNKNOWN) { + if (initializer == WEIGHT_UNKNOWN) { ml_logw("Warning: Weight Initalization Type is not set. " "WEIGHT_XAVIER_NORMAL is used by default"); - init_type = WEIGHT_XAVIER_NORMAL; + initializer = WEIGHT_XAVIER_NORMAL; } - switch (init_type) { + switch (initializer) { + case WEIGHT_ZEROS: + w.setZero(); + break; case WEIGHT_LECUN_NORMAL: w.setRandNormal(0.0f, sqrt(1.0f / w_dim.height())); break; @@ -183,12 +186,6 @@ void Layer::setProperty(const PropertyType type, const std::string &value) { input_dim.batch(batch_size); } break; - case PropertyType::bias_init_zero: - if (!value.empty()) { - status = setBoolean(this->bias_init_zero, value); - throw_status(status); - } - break; case PropertyType::activation: if (!value.empty()) { status = setActivation((ActiType)parseType(value, TOKEN_ACTI)); @@ -215,9 +212,15 @@ void Layer::setProperty(const PropertyType type, const std::string &value) { throw_status(status); } break; - case PropertyType::weight_ini: + case PropertyType::weight_initializer: + if (!value.empty()) { + weight_initializer = + (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT); + } + break; + case PropertyType::bias_initializer: if (!value.empty()) { - weight_ini_type = (WeightIniType)parseType(value, TOKEN_WEIGHTINI); + bias_initializer = (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT); } break; default: @@ -262,7 +265,6 @@ void Layer::printPropertiesMeta(std::ostream &out) { void Layer::printProperties(std::ostream &out) { out << "Trainable: " << trainable << std::endl; - printIfValid(out, PropertyType::bias_init_zero, bias_init_zero); printIfValid(out, PropertyType::weight_decay, static_cast(weight_decay.type)); printIfValid(out, PropertyType::weight_decay_lambda, weight_decay.lambda); diff --git a/nntrainer/src/neuralnet.cpp b/nntrainer/src/neuralnet.cpp index 7f640a0..31a2891 100644 --- a/nntrainer/src/neuralnet.cpp +++ b/nntrainer/src/neuralnet.cpp @@ -42,7 +42,7 @@ NeuralNetwork::NeuralNetwork() : epoch(1), loss(0.0f), cost(COST_UNKNOWN), - weight_ini(WEIGHT_UNKNOWN), + weight_initializer(WEIGHT_UNKNOWN), net_type(NET_UNKNOWN), data_buffer(NULL), continue_train(false), diff --git a/nntrainer/src/parse_util.cpp b/nntrainer/src/parse_util.cpp index 55025d5..f9c8898 100644 --- a/nntrainer/src/parse_util.cpp +++ b/nntrainer/src/parse_util.cpp @@ -114,6 +114,7 @@ unsigned int parseType(std::string ll, InputType t) { /** * @brief Weight Initialization Type String from configure file + * "zeros" : Zero Initialization * "lecun_normal" : LeCun Normal Initialization * "lecun_uniform" : LeCun Uniform Initialization * "xavier_normal" : Xavier Normal Initialization @@ -121,9 +122,9 @@ unsigned int parseType(std::string ll, InputType t) { * "he_normal" : He Normal Initialization * "he_uniform" : He Uniform Initialization */ - std::array weight_ini_string = { - "lecun_normal", "lecun_uniform", "xavier_normal", - "xavier_uniform", "he_normal", "he_uniform"}; + std::array weight_ini_string = { + "zeros", "lecun_normal", "lecun_uniform", "xavier_normal", + "xavier_uniform", "he_normal", "he_uniform"}; /** * @brief Weight Decay String from configure file @@ -199,14 +200,14 @@ unsigned int parseType(std::string ll, InputType t) { } ret = (unsigned int)LayerType::LAYER_UNKNOWN; break; - case TOKEN_WEIGHTINI: + case TOKEN_WEIGHT_INIT: for (i = 0; i < weight_ini_string.size(); i++) { if (!strncasecmp(weight_ini_string[i].c_str(), ll.c_str(), weight_ini_string[i].size())) { return (i); } } - ret = (unsigned int)WeightIniType::WEIGHT_UNKNOWN; + ret = (unsigned int)WeightInitializer::WEIGHT_UNKNOWN; break; case TOKEN_WEIGHT_DECAY: for (i = 0; i < weight_decay_string.size(); i++) { @@ -246,16 +247,16 @@ unsigned int parseType(std::string ll, InputType t) { /** * @brief Layer Properties - * input_shape = 0, - * bias_init_zero = 1, - * normalization = 2, - * standardization = 3, - * activation = 4, - * epsilon = 5 - * weight_decay = 6 - * weight_decay_lambda = 7 - * unit = 8 - * weight_ini = 9 + * input_shape = 0 + * normalization = 1 + * standardization = 2 + * activation = 3 + * epsilon = 4 + * weight_decay = 5 + * weight_decay_lambda = 6 + * unit = 7 + * weight_initializer = 8 + * bias_initializer = 9 * filter = 10 * kernel_size = 11 * stride = 12 @@ -274,13 +275,28 @@ unsigned int parseType(std::string ll, InputType t) { * Pooling2DLayer has 12, 13, 14, 15 properties. * BatchNormalizationLayer has 0, 1, 5, 6, 7 properties. */ -static std::array property_string = { - "input_shape", "bias_init_zero", "normalization", "standardization", - "activation", "epsilon", "weight_decay", "weight_decay_lambda", - "unit", "weight_ini", "filter", "kernel_size", - "stride", "padding", "pooling_size", "pooling", - "flatten", "name", "num_inputs", "num_outputs", - "batch_size", "unknown"}; +static std::array property_string = {"input_shape", + "normalization", + "standardization", + "activation", + "epsilon", + "weight_decay", + "weight_decay_lambda", + "unit", + "weight_initializer", + "bias_initializer", + "filter", + "kernel_size", + "stride", + "padding", + "pooling_size", + "pooling", + "flatten", + "name", + "num_inputs", + "num_outputs", + "batch_size", + "unknown"}; unsigned int parseLayerProperty(std::string property) { unsigned int i; diff --git a/test/include/nntrainer_test_util.h b/test/include/nntrainer_test_util.h index 135c7c1..682242d 100644 --- a/test/include/nntrainer_test_util.h +++ b/test/include/nntrainer_test_util.h @@ -216,7 +216,7 @@ const std::string config_str = "[Model]" "\n" "Input_Shape = 1:1:62720" "\n" - "bias_init_zero = true" + "bias_initializer = zeros" "\n" "Normalization = true" "\n" @@ -228,7 +228,7 @@ const std::string config_str = "[Model]" "\n" "Unit = 10" "\n" - "bias_init_zero = true" + "bias_initializer = zeros" "\n" "Activation = softmax" "\n"; @@ -279,7 +279,7 @@ const std::string config_str2 = "[Model]" "\n" "Input_Shape = 3:28:28" "\n" - "bias_init_zero = true" + "bias_initializer = zeros" "\n" "Activation = sigmoid" "\n" @@ -295,7 +295,7 @@ const std::string config_str2 = "[Model]" "\n" "padding=0,0" "\n" - "weight_ini=xavier_uniform" + "weight_initializer=xavier_uniform" "\n" "flatten = false" "\n" @@ -305,7 +305,7 @@ const std::string config_str2 = "[Model]" "\n" "Unit = 10" "\n" - "bias_init_zero = true" + "bias_initializer = zeros" "\n" "Activation = softmax" "\n"; diff --git a/test/tizen_capi/test_conf.ini b/test/tizen_capi/test_conf.ini index cf9f6a7..34f09bd 100644 --- a/test/tizen_capi/test_conf.ini +++ b/test/tizen_capi/test_conf.ini @@ -22,17 +22,17 @@ LabelData = "label.dat" [inputlayer] Type = input HiddenSize = 128 # Input Layer Dimension -Bias_init_zero = true # Zero Bias +Bias_initializer = zeros # Zero Bias Activation = sigmoid [fc1layer] Type = fully_connected HiddenSize = 20 # Hidden Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = sigmoid [outputlayer] Type = output HiddenSize = 3 # Output Layer Dimension ( = Weight Width ) -Bias_init_zero = true +Bias_initializer = zeros Activation = sigmoid diff --git a/test/tizen_capi/unittest_tizen_capi.cpp b/test/tizen_capi/unittest_tizen_capi.cpp index 6f4349c..05cccbc 100644 --- a/test/tizen_capi/unittest_tizen_capi.cpp +++ b/test/tizen_capi/unittest_tizen_capi.cpp @@ -127,7 +127,7 @@ TEST(nntrainer_capi_nnmodel, compile_05_p) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[0]); @@ -142,9 +142,9 @@ TEST(nntrainer_capi_nnmodel, compile_05_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", - "weight_ini=xavier_uniform", "name=fc100", NULL); + "weight_initializer=xavier_uniform", "name=fc100", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[1]); @@ -191,7 +191,7 @@ TEST(nntrainer_capi_nnmodel, compile_06_n) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); /** Find layer before adding */ @@ -403,7 +403,7 @@ TEST(nntrainer_capi_nnmodel, addLayer_04_p) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[0]); @@ -413,7 +413,7 @@ TEST(nntrainer_capi_nnmodel, addLayer_04_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL); EXPECT_EQ(status, ML_ERROR_NONE); @@ -454,7 +454,7 @@ TEST(nntrainer_capi_nnmodel, addLayer_05_n) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layer, "unit= 10", "activation=softmax", "bias_init_zero=true", + layer, "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL); EXPECT_EQ(status, ML_ERROR_NONE); @@ -513,7 +513,7 @@ TEST(nntrainer_capi_nnmodel, create_optimizer_02_p) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[0]); @@ -523,7 +523,7 @@ TEST(nntrainer_capi_nnmodel, create_optimizer_02_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL); EXPECT_EQ(status, ML_ERROR_NONE); @@ -563,7 +563,7 @@ TEST(nntrainer_capi_nnmodel, create_optimizer_03_p) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[0]); @@ -573,7 +573,7 @@ TEST(nntrainer_capi_nnmodel, create_optimizer_03_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", NULL); EXPECT_EQ(status, ML_ERROR_NONE); @@ -617,7 +617,7 @@ TEST(nntrainer_capi_nnmodel, train_with_file_01_p) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[0]); @@ -627,9 +627,9 @@ TEST(nntrainer_capi_nnmodel, train_with_file_01_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", - "weight_ini=xavier_uniform", NULL); + "weight_initializer=xavier_uniform", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[1]); @@ -686,7 +686,7 @@ TEST(nntrainer_capi_nnmodel, train_with_generator_01_p) { status = ml_train_layer_set_property(layers[0], "input_shape=1:1:62720", "normalization=true", - "bias_init_zero=true", NULL); + "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[0]); @@ -696,9 +696,9 @@ TEST(nntrainer_capi_nnmodel, train_with_generator_01_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property( - layers[1], "unit= 10", "activation=softmax", "bias_init_zero=true", + layers[1], "unit= 10", "activation=softmax", "bias_initializer=zeros", "weight_decay=l2norm", "weight_decay_lambda=0.005", - "weight_ini=xavier_uniform", NULL); + "weight_initializer=xavier_uniform", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_add_layer(model, layers[1]); diff --git a/test/tizen_capi/unittest_tizen_capi_layer.cpp b/test/tizen_capi/unittest_tizen_capi_layer.cpp index 83cadbb..6e2dcc5 100644 --- a/test/tizen_capi/unittest_tizen_capi_layer.cpp +++ b/test/tizen_capi/unittest_tizen_capi_layer.cpp @@ -89,7 +89,7 @@ TEST(nntrainer_capi_nnlayer, setproperty_02_p) { status = ml_train_layer_set_property(handle, "unit=10", NULL); EXPECT_EQ(status, ML_ERROR_NONE); - status = ml_train_layer_set_property(handle, "bias_init_zero=true", NULL); + status = ml_train_layer_set_property(handle, "bias_initializer=zeros", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_layer_set_property(handle, "activation =sigmoid", NULL); diff --git a/test/unittest/unittest_nntrainer_internal.cpp b/test/unittest/unittest_nntrainer_internal.cpp index 0fa421b..33b0141 100644 --- a/test/unittest/unittest_nntrainer_internal.cpp +++ b/test/unittest/unittest_nntrainer_internal.cpp @@ -132,8 +132,8 @@ TEST(nntrainer_NeuralNetwork, load_config_06_p) { TEST(nntrainer_NeuralNetwork, load_config_07_p) { int status = ML_ERROR_NONE; RESET_CONFIG("./test.ini"); - replaceString("bias_init_zero = true", "Bias_Init_Zero = false", "./test.ini", - config_str); + replaceString("bias_initializer = zeros", "Bias_Initializer = he_normal", + "./test.ini", config_str); nntrainer::NeuralNetwork NN; status = NN.loadFromConfig("./test.ini"); diff --git a/test/unittest/unittest_nntrainer_layers.cpp b/test/unittest/unittest_nntrainer_layers.cpp index efbe5c1..58f2663 100644 --- a/test/unittest/unittest_nntrainer_layers.cpp +++ b/test/unittest/unittest_nntrainer_layers.cpp @@ -527,7 +527,7 @@ protected: setInputDim("1:1:12"); setProperty("batch_size=3"); setProperty("unit=15"); - setProperty("bias_init_zero=true"); + setProperty("bias_initializer=zeros"); } void matchUpdatedWeightsGradients() { @@ -853,11 +853,11 @@ protected: virtual void prepareLayer() { int status = setProperty("input_shape=3:28:28 | batch_size=32 |" - "bias_init_zero=true |" + "bias_initializer=zeros |" "activation=sigmoid |" "weight_decay=l2norm |" "weight_decay_lambda= 0.005 |" - "weight_ini=xavier_uniform |" + "weight_initializer=xavier_uniform |" "normalization=true |" "filter=12 | kernel_size= 5,5 | stride=3,3 | padding=1,1"); @@ -918,8 +918,8 @@ TEST_F(nntrainer_Conv2DLayer, save_read_01_p) { */ TEST_F(nntrainer_Conv2DLayer, forwarding_01_p) { reinitialize("input_shape=3:7:7 | batch_size=1 |" - "bias_init_zero = true |" - "weight_ini=xavier_uniform |" + "bias_initializer = zeros |" + "weight_initializer=xavier_uniform |" "filter=2 | kernel_size=3,3 | stride=1, 1 | padding=0,0"); ASSERT_EQ(in.getDim(), nntrainer::TensorDim(1, 3, 7, 7)); @@ -938,8 +938,8 @@ TEST_F(nntrainer_Conv2DLayer, forwarding_01_p) { TEST_F(nntrainer_Conv2DLayer, forwarding_02_p) { reinitialize("input_shape=3:7:7 | batch_size=2 |" - "bias_init_zero = true |" - "weight_ini=xavier_uniform |" + "bias_initializer = zeros |" + "weight_initializer=xavier_uniform |" "filter=3 | kernel_size=3,3 | stride=1, 1 | padding=0,0"); ASSERT_EQ(in.getDim(), nntrainer::TensorDim(2, 3, 7, 7)); @@ -954,8 +954,8 @@ TEST_F(nntrainer_Conv2DLayer, forwarding_02_p) { TEST_F(nntrainer_Conv2DLayer, backwarding_01_p) { status = reinitialize("input_shape=3:7:7 | batch_size=1 |" - "bias_init_zero=true |" - "weight_ini=xavier_uniform |" + "bias_initializer=zeros |" + "weight_initializer=xavier_uniform |" "filter=2 |" "kernel_size= 3,3 |" "stride=1, 1 |" @@ -1007,8 +1007,8 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_01_p) { TEST_F(nntrainer_Conv2DLayer, backwarding_02_p) { status = reinitialize("input_shape=3:7:7 | batch_size=2 |" - "bias_init_zero=true |" - "weight_ini=xavier_uniform |" + "bias_initializer=zeros |" + "weight_initializer=xavier_uniform |" "filter=3 |" "kernel_size= 3,3 |" "stride=1, 1 |" diff --git a/test/unittest/unittest_nntrainer_modelfile.cpp b/test/unittest/unittest_nntrainer_modelfile.cpp index 15e766d..c79578f 100644 --- a/test/unittest/unittest_nntrainer_modelfile.cpp +++ b/test/unittest/unittest_nntrainer_modelfile.cpp @@ -110,7 +110,7 @@ static IniSection flatten("flat", "Type = flatten"); static IniSection input("inputlayer", "Type = input |" "Input_Shape = 1:1:62720 |" - "bias_init_zero = true |" + "bias_initializer = zeros |" "Normalization = true |" "Activation = sigmoid"); @@ -119,11 +119,11 @@ static IniSection act_relu("activation_relu", "Type = activation | " static IniSection out("fclayer", "Type = fully_connected |" "Unit = 10 |" - "bias_init_zero = true |" + "bias_initializer = zeros |" "Activation = softmax"); static IniSection conv2d("conv2d", "Type = conv2d |" - "bias_init_zero = true |" + "bias_initializer = zeros |" "Activation = sigmoid |" "filter = 6 |" "kernel_size = 5,5 |" -- 2.7.4