Output Layer Can be replaced by Fully Connected Layer.
No need to be exist.
**Self evaluation:**
1. Build test: [X]Passed [ ]Failed [ ]Skipped
2. Run test: [X]Passed [ ]Failed [ ]Skipped
Signed-off-by: jijoong.moon <jijoong.moon@samsung.com>
featureVector.resize(feature_size);
getFeature(img, featureVector);
nntrainer::Tensor X = nntrainer::Tensor({featureVector});
- cout << NN.forwarding(X).apply(stepFunction) << endl;
+ cout << NN.forwarding(X, status).apply(stepFunction) << endl;
}
/**
* @brief Finalize NN
# Layer Section : Name
[inputlayer]
Type = input
-Id = 0 # Layer Id
HiddenSize = 62720 # Input Layer Dimension
Bias_zero = true # Zero Bias
Normalization = true
Activation = sigmoid # activation : sigmoid, tanh
[outputlayer]
-Type = output
-Id = 1
+Type = fully_connected
HiddenSize = 10 # Output Layer Dimension ( = Weight Width )
Bias_zero = true
Activation = softmax # activation : sigmoid, softmax
# Layer Section : Name
[inputlayer]
Type = input
-Id = 0 # Layer Id
HiddenSize = 62720 # Input Layer Dimension
Bias_zero = true # Zero Bias
Normalization = true
Activation = sigmoid # activation : sigmoid, tanh
[outputlayer]
-Type = output
-Id = 1
+Type = fully_connected
HiddenSize = 10 # Output Layer Dimension ( = Weight Width )
Bias_zero = true
Activation = softmax # activation : sigmoid, softmax
* @brief forward propagation
*/
int cn = 0;
+ int status = 0;
for (unsigned int j = 0; j < inputVector.size(); ++j) {
std::vector<std::vector<float>> in, label;
in.push_back(inputVector[j]);
label.push_back(outputVector[j]);
- if (NN.forwarding(nntrainer::Tensor(in))
+ if (NN.forwarding(nntrainer::Tensor(in), status)
.apply(stepFunction)
.getValue(0, 0, 0) == label[0][0])
cn++;
# Layer Section : Name
[inputlayer]
Type = input
-Id = 0 # Layer Id
HiddenSize = 2
Bias_zero = true # Zero Bias
Activation = sigmoid
[outputlayer]
-Type = output
-Id = 1
+Type = fully_connected
HiddenSize = 1
Bias_zero = true
-Activation = sigmoid
\ No newline at end of file
+Activation = sigmoid
# Layer Section : Name
[inputlayer]
Type = input
-Id = 0 # Layer Id
HiddenSize = 4 # Input Layer Dimension
Bias_zero = true # Zero Bias
Activation = tanh # activation : sigmoid, tanh
[fc1layer]
Type = fully_connected
-Id = 1
HiddenSize = 50 # Hidden Layer Dimension ( = Weight Width )
Bias_zero = true
Activation = tanh # activation : sigmoid, tanh
[fc2layer]
Type = fully_connected
-Id = 2
HiddenSize = 50
Bias_zero = true
Activation = tanh # activation : sigmoid, tanh
[outputlayer]
-Type = output
-Id = 3
+Type = fully_connected
HiddenSize = 2 # Output Layer Dimension ( = Weight Width )
Bias_zero = true
-Activation = sigmoid # activation : sigmoid, tanh
\ No newline at end of file
+Activation = sigmoid # activation : sigmoid, tanh
std::deque<Experience> expQ;
PTR env;
+ int status = 0;
/**
* @brief Initialize Environment
/**
* @brief get action with input State with mainNet
*/
- nntrainer::Tensor test = mainNet.forwarding(nntrainer::Tensor({input}));
+ nntrainer::Tensor test =
+ mainNet.forwarding(nntrainer::Tensor({input}), status);
std::vector<float> temp = test.mat2vec();
action.push_back(argmax(temp));
/**
* @brief run forward propagation with mainNet
*/
- nntrainer::Tensor Q = mainNet.forwarding(nntrainer::Tensor(inbatch));
+ nntrainer::Tensor Q =
+ mainNet.forwarding(nntrainer::Tensor(inbatch), status);
/**
* @brief run forward propagation with targetNet
*/
nntrainer::Tensor NQ =
- targetNet.forwarding(nntrainer::Tensor(next_inbatch));
+ targetNet.forwarding(nntrainer::Tensor(next_inbatch), status);
std::vector<float> nqa = NQ.mat2vec();
/**
srand(time(NULL));
std::string ini_file = data_path + "ini.bin";
std::vector<std::vector<float>> inputVector, outputVector;
-
+ int status = 0;
/**
* @brief Extract Feature
*/
featureVector.resize(128);
getFeature(img, featureVector);
nntrainer::Tensor X = nntrainer::Tensor({featureVector});
- cout << NN.forwarding(X).apply(stepFunction) << endl;
+ cout << NN.forwarding(X, status).apply(stepFunction) << endl;
}
/**
# Layer Section : Name
[inputlayer]
Type = input
-Id = 0 # Layer Id
HiddenSize = 128 # Input Layer Dimension
Bias_zero = true # Zero Bias
Activation = sigmoid
[fc1layer]
Type = fully_connected
-Id = 1
HiddenSize = 20 # Hidden Layer Dimension ( = Weight Width )
Bias_zero = true
Activation = sigmoid
[outputlayer]
-Type = output
-Id = 3
+Type = fully_connected
HiddenSize = 3 # Output Layer Dimension ( = Weight Width )
Bias_zero = true
Activation = sigmoid
* @brief Enumeration of layer type
* 0. Input Layer type
* 1. Fully Connected Layer type
- * 2. Output Layer type
- * 3. Unknown
+ * 2. Unknown
*/
-typedef enum {
- LAYER_IN,
- LAYER_FC,
- LAYER_OUT,
- LAYER_BN,
- LAYER_UNKNOWN
-} LayerType;
+typedef enum { LAYER_IN, LAYER_FC, LAYER_BN, LAYER_UNKNOWN } LayerType;
/**
* @brief Enumeration of Weight Initialization Type
* @param[in] in Input Tensor taken by upper layer
* @retval Output Tensor
*/
- virtual Tensor forwarding(Tensor in) = 0;
+ virtual Tensor forwarding(Tensor in, int &status) = 0;
/**
* @brief Forward Propation of neural Network
* @param[in] in Input Tensor taken by upper layer
* @retval Output Tensor
*/
- virtual Tensor forwarding(Tensor in, Tensor output) = 0;
+ virtual Tensor forwarding(Tensor in, Tensor output, int &status) = 0;
/**
* @brief Back Propation of neural Network
* @param[in] b batch
* @param[in] h Height
* @param[in] w Width
- * @param[in] id index of this layer
+ * @param[in] last last layer
* @param[in] init_zero Bias initialization with zero
* @param[in] wini Weight Initialization Scheme
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- virtual int initialize(int b, int h, int w, int id, bool init_zero,
+ virtual int initialize(int b, int h, int w, bool last, bool init_zero,
WeightIniType wini) = 0;
/**
int checkValidation();
+protected:
/**
* @brief Input Tensor
*/
Tensor hidden;
/**
- * @brief Layer index
+ * @brief last layer
*/
- unsigned int index;
+ bool last_layer;
/**
* @brief batch size of Weight Data
* @param[in] in input Tensor from lower layer.
* @retval return Input Tensor
*/
- Tensor forwarding(Tensor in);
+ Tensor forwarding(Tensor in, int &status);
/**
* @brief foward propagation : return Input Tensor
* @param[in] output label Tensor.
* @retval return Input Tensor
*/
- Tensor forwarding(Tensor in, Tensor output) { return forwarding(in); };
+ Tensor forwarding(Tensor in, Tensor output, int &status) {
+ return forwarding(in, status);
+ };
/**
* @brief Set Optimizer
* @param[in] b batch size
* @param[in] h height
* @param[in] w width
- * @param[in] id index of this layer
+ * @param[in] last last layer
* @param[in] init_zero boolean to set Bias zero
* @param[in] wini Weight Initialization Scheme
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int initialize(int b, int h, int w, int id, bool init_zero,
+ int initialize(int b, int h, int w, bool last, bool init_zero,
WeightIniType wini);
/**
/**
* @brief Constructor of Fully Connected Layer
*/
- FullyConnectedLayer(){};
+ FullyConnectedLayer() : loss(0.0), cost(COST_UNKNOWN){};
/**
* @brief Destructor of Fully Connected Layer
* @param[in] in Input Tensor from upper layer
* @retval Activation(W x input + B)
*/
- Tensor forwarding(Tensor in);
+ Tensor forwarding(Tensor in, int &status);
/**
* @brief foward propagation : return Input Tensor
* @param[in] output label Tensor.
* @retval Activation(W x input + B)
*/
- Tensor forwarding(Tensor in, Tensor output) { return forwarding(in); };
+ Tensor forwarding(Tensor in, Tensor output, int &status);
/**
* @brief back propagation
* @param[in] b batch size
* @param[in] h height
* @param[in] w width
- * @param[in] id layer index
- * @param[in] init_zero boolean to set Bias zero
- * @param[in] wini Weight Initialization Scheme
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize(int b, int h, int w, int id, bool init_zero,
- WeightIniType wini);
-
-private:
- Tensor weight;
- Tensor bias;
-};
-
-/**
- * @class OutputLayer
- * @brief OutputLayer (has Cost Function & Weight, Bias)
- */
-class OutputLayer : public Layer {
-public:
- /**
- * @brief Constructor of OutputLayer
- */
- OutputLayer() : loss(0.0), cost(COST_UNKNOWN){};
-
- /**
- * @brief Destructor of OutputLayer
- */
- ~OutputLayer(){};
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file);
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &flle);
-
- /**
- * @brief forward propagation with input
- * @param[in] in Input Tensor from upper layer
- * @retval Activation(W x input + B)
- */
- Tensor forwarding(Tensor in);
-
- /**
- * @brief forward propagation with input and set loss
- * @param[in] in Input Tensor from upper layer
- * @param[in] output Label Tensor
- * @retval Activation(W x input + B)
- */
- Tensor forwarding(Tensor in, Tensor output);
-
- /**
- * @brief back propagation
- * Calculate dJdB & dJdW & Update W & B
- * @param[in] input Input Tensor from lower layer
- * @param[in] iteration Number of Epoch for ADAM
- * @retval dJdB x W Tensor
- */
- Tensor backwarding(Tensor label, int iteration);
-
- /**
- * @brief initialize layer
- * @param[in] b batch size
- * @param[in] h height
- * @param[in] w width
- * @param[in] id layer index
+ * @param[in] last last layer
* @param[in] init_zero boolean to set Bias zero
* @param[in] wini Weight Initialization Scheme
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int initialize(int b, int w, int h, int id, bool init_zero,
+ int initialize(int b, int h, int w, bool last, bool init_zero,
WeightIniType wini);
-
/**
* @brief get Loss value
*/
*/
int setCost(CostType c);
- /**
- * @brief copy layer
- * @param[in] l layer to copy
- */
- void copy(Layer *l);
-
private:
Tensor weight;
Tensor bias;
* @param[in] in Input Tensor from upper layer
* @retval Activation(W x input + B)
*/
- Tensor forwarding(Tensor in);
+ Tensor forwarding(Tensor in, int &status);
/**
* @brief foward propagation : return Input Tensor
* @param[in] output label Tensor.
* @retval Activation(W x input + B)
*/
- Tensor forwarding(Tensor in, Tensor output) { return forwarding(in); };
+ Tensor forwarding(Tensor in, Tensor output, int &status) {
+ return forwarding(in, status);
+ };
/**
* @brief back propagation
* @param[in] b batch size
* @param[in] h height
* @param[in] w width
- * @param[in] id layer index
+ * @param[in] last last layer
* @param[in] init_zero boolean to set Bias zero
* @param[in] wini Weight Initialization Scheme
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int initialize(int b, int h, int w, int id, bool init_zero,
+ int initialize(int b, int h, int w, bool last, bool init_zero,
WeightIniType wini);
private:
* @param[in] input Input Tensor X
* @retval Output Tensor Y
*/
- Tensor forwarding(Tensor input);
+ Tensor forwarding(Tensor input, int &status);
/**
* @brief forward propagation
* @param[in] label Input Tensor Y2
* @retval Output Tensor Y
*/
- Tensor forwarding(Tensor input, Tensor output);
+ Tensor forwarding(Tensor input, Tensor output, int &status);
/**
* @brief back propagation to update W & B
* @param[in] expectedOutput Lable Tensor Y
* @param[in] iteration Epoch Number for ADAM
*/
- void backwarding(Tensor input, Tensor expected_output, int iteration);
+ int backwarding(Tensor input, Tensor expected_output, int iteration);
/**
* @brief save W & B into file
Layer::Layer() {
type = LAYER_UNKNOWN;
activation_type = ACT_UNKNOWN;
- index = 0;
+ last_layer = false;
batch = 0;
width = 0;
height = 0;
void InputLayer::copy(Layer *l) {
InputLayer *from = static_cast<InputLayer *>(l);
this->opt = from->opt;
- this->index = from->index;
+ this->last_layer = from->last_layer;
this->height = from->height;
this->width = from->width;
this->input.copy(from->input);
this->hidden.copy(from->hidden);
}
-Tensor InputLayer::forwarding(Tensor in) {
+Tensor InputLayer::forwarding(Tensor in, int &status) {
input = in;
if (normalization)
input = input.normalization();
return input;
}
-int InputLayer::initialize(int b, int h, int w, int id, bool init_zero,
+int InputLayer::initialize(int b, int h, int w, bool last, bool init_zero,
WeightIniType wini) {
int status = ML_ERROR_NONE;
if (b <= 0 || h <= 0 || w <= 0) {
this->batch = b;
this->width = w;
this->height = h;
- this->index = 0;
+ this->last_layer = last;
this->bn_fallow = false;
return status;
}
-int FullyConnectedLayer::initialize(int b, int h, int w, int id, bool init_zero,
- WeightIniType wini) {
+int FullyConnectedLayer::initialize(int b, int h, int w, bool last,
+ bool init_zero, WeightIniType wini) {
int status = ML_ERROR_NONE;
if (b <= 0 || h <= 0 || w <= 0) {
ml_loge("Error: Dimension must be greater than 0");
this->batch = b;
this->width = w;
this->height = h;
- this->index = id;
+ this->last_layer = last;
this->init_zero = init_zero;
this->bn_fallow = false;
return status;
}
-Tensor FullyConnectedLayer::forwarding(Tensor in) {
- input = in;
- hidden = input.dot(weight).add(bias);
-
- if (this->bn_fallow)
- return hidden;
-
- return hidden.apply(activation);
- ;
-}
-
-void FullyConnectedLayer::read(std::ifstream &file) {
- weight.read(file);
- bias.read(file);
-}
-
-void FullyConnectedLayer::save(std::ofstream &file) {
- weight.save(file);
- bias.save(file);
-}
-
-void FullyConnectedLayer::copy(Layer *l) {
- FullyConnectedLayer *from = static_cast<FullyConnectedLayer *>(l);
- this->opt = from->opt;
- this->index = from->index;
- this->height = from->height;
- this->width = from->width;
- this->input.copy(from->input);
- this->hidden.copy(from->hidden);
- this->weight.copy(from->weight);
- this->bias.copy(from->bias);
-}
-
-Tensor FullyConnectedLayer::backwarding(Tensor derivative, int iteration) {
- Tensor djdb;
-
- djdb = derivative.multiply(hidden.apply(activation_prime));
-
- Tensor ret = djdb.dot(weight.transpose());
-
- Tensor djdw = input.transpose().dot(djdb);
-
- opt.calculate(djdw, djdb, weight, bias, iteration, this->init_zero);
-
- return ret;
-}
-
-int OutputLayer::initialize(int b, int h, int w, int id, bool init_zero,
- WeightIniType wini) {
- int status = ML_ERROR_NONE;
- if (b <= 0 || h <= 0 || w <= 0) {
- ml_loge("Error: Dimension must be greater than 0");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- this->batch = b;
- this->width = w;
- this->height = h;
- this->index = id;
- this->init_zero = init_zero;
-
- bias = Tensor(1, w);
- this->bn_fallow = false;
-
- weight = weightInitialization(w, h, wini, status);
-
- if (init_zero) {
- bias.setZero();
- } else {
- bias = bias.apply(random);
- }
- return status;
-}
-
-int OutputLayer::setCost(CostType c) {
+int FullyConnectedLayer::setCost(CostType c) {
int status = ML_ERROR_NONE;
if (c == COST_UNKNOWN) {
ml_loge("Error: Unknown cost fucntion");
return status;
}
-Tensor OutputLayer::forwarding(Tensor in) {
+Tensor FullyConnectedLayer::forwarding(Tensor in, int &status) {
input = in;
hidden = input.dot(weight).add(bias);
+
+ if (this->bn_fallow)
+ return hidden;
+
if (activation_type == ACT_SOFTMAX) {
return hidden.apply(softmax);
} else {
}
}
-Tensor OutputLayer::forwarding(Tensor in, Tensor output) {
+Tensor FullyConnectedLayer::forwarding(Tensor in, Tensor output, int &status) {
+ if (!this->last_layer) {
+ ml_loge("Error: Cannot update cost. This is not last layer of network");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
input = in;
hidden = input.dot(weight).add(bias);
Tensor y2 = output;
return y;
}
-void OutputLayer::read(std::ifstream &file) {
+void FullyConnectedLayer::read(std::ifstream &file) {
weight.read(file);
bias.read(file);
}
-void OutputLayer::save(std::ofstream &file) {
+void FullyConnectedLayer::save(std::ofstream &file) {
weight.save(file);
bias.save(file);
}
-void OutputLayer::copy(Layer *l) {
- OutputLayer *from = static_cast<OutputLayer *>(l);
+void FullyConnectedLayer::copy(Layer *l) {
+ FullyConnectedLayer *from = static_cast<FullyConnectedLayer *>(l);
this->opt = from->opt;
- this->index = from->index;
+ this->last_layer = from->last_layer;
this->height = from->height;
this->width = from->width;
this->input.copy(from->input);
this->loss = from->loss;
}
-Tensor OutputLayer::backwarding(Tensor label, int iteration) {
+Tensor FullyConnectedLayer::backwarding(Tensor derivative, int iteration) {
+ Tensor djdb;
float loss_sum = 0.0;
- Tensor y2 = label;
+ Tensor y2 = derivative;
Tensor y;
- if (activation_type == ACT_SOFTMAX)
- y = hidden.apply(softmax);
- else
- y = hidden.apply(activation);
-
- Tensor ret;
- Tensor djdb;
-
- float ll = opt.getLearningRate();
- if (opt.getDecaySteps() != -1) {
- ll = ll * pow(opt.getDecayRate(), (iteration / opt.getDecaySteps()));
- }
-
- switch (cost) {
- case COST_MSR: {
- Tensor sub = y2.subtract(y);
- Tensor l = (sub.multiply(sub)).sum().multiply(0.5);
- std::vector<float> t = l.mat2vec();
- for (int i = 0; i < l.getBatch(); i++) {
- loss_sum += t[i];
- }
-
- loss = loss_sum / (float)l.getBatch();
- if (opt.getWeightDecayType() == WeightDecayType::l2norm) {
- loss += opt.getWeightDecayLambda() * 0.5 * (weight.l2norm());
- }
- if (activation_type == ACT_SOFTMAX) {
- djdb = y.subtract(y2).multiply(y.apply(softmaxPrime));
- } else {
- djdb = y.subtract(y2).multiply(hidden.apply(activation_prime));
- }
- } break;
- case COST_ENTROPY: {
- Tensor l;
- if (activation_type == ACT_SIGMOID) {
- djdb = y.subtract(y2).multiply(1.0 / y.getWidth());
- l = (y2.multiply(y.apply(logFloat))
- .add((y2.multiply(-1.0).add(1.0))
- .multiply((y.multiply(-1.0).add(1.0)).apply(logFloat))))
- .multiply(-1.0 / (y2.getWidth()))
- .sum();
- } else if (activation_type == ACT_SOFTMAX) {
- djdb = y.subtract(y2).multiply(1.0 / y.getWidth());
- l =
- (y2.multiply(y.apply(logFloat))).multiply(-1.0 / (y2.getWidth())).sum();
- } else {
- ml_loge("Only support sigmoid & softmax for cross entropy loss");
- exit(0);
- }
- std::vector<float> t = l.mat2vec();
-
- for (int i = 0; i < l.getBatch(); i++) {
- loss_sum += t[i];
+ if (!last_layer) {
+ djdb = derivative.multiply(hidden.apply(activation_prime));
+ } else {
+ if (activation_type == ACT_SOFTMAX)
+ y = hidden.apply(softmax);
+ else
+ y = hidden.apply(activation);
+ float ll = opt.getLearningRate();
+ if (opt.getDecaySteps() != -1) {
+ ll = ll * pow(opt.getDecayRate(), (iteration / opt.getDecaySteps()));
}
- loss = loss_sum / (float)l.getBatch();
- if (opt.getWeightDecayType() == WeightDecayType::l2norm) {
- loss += opt.getWeightDecayLambda() * 0.5 * (weight.l2norm());
+ switch (cost) {
+ case COST_MSR: {
+ Tensor sub = y2.subtract(y);
+ Tensor l = (sub.multiply(sub)).sum().multiply(0.5);
+ std::vector<float> t = l.mat2vec();
+ for (int i = 0; i < l.getBatch(); i++) {
+ loss_sum += t[i];
+ }
+
+ loss = loss_sum / (float)l.getBatch();
+ if (opt.getWeightDecayType() == WeightDecayType::l2norm) {
+ loss += opt.getWeightDecayLambda() * 0.5 * (weight.l2norm());
+ }
+ if (activation_type == ACT_SOFTMAX) {
+ djdb = y.subtract(y2).multiply(y.apply(softmaxPrime));
+ } else {
+ djdb = y.subtract(y2).multiply(hidden.apply(activation_prime));
+ }
+ } break;
+ case COST_ENTROPY: {
+ Tensor l;
+ if (activation_type == ACT_SIGMOID) {
+ djdb = y.subtract(y2).multiply(1.0 / y.getWidth());
+ l = (y2.multiply(y.apply(logFloat))
+ .add((y2.multiply(-1.0).add(1.0))
+ .multiply((y.multiply(-1.0).add(1.0)).apply(logFloat))))
+ .multiply(-1.0 / (y2.getWidth()))
+ .sum();
+ } else if (activation_type == ACT_SOFTMAX) {
+ djdb = y.subtract(y2).multiply(1.0 / y.getWidth());
+ l = (y2.multiply(y.apply(logFloat)))
+ .multiply(-1.0 / (y2.getWidth()))
+ .sum();
+ } else {
+ ml_loge("Only support sigmoid & softmax for cross entropy loss");
+ exit(0);
+ }
+
+ std::vector<float> t = l.mat2vec();
+
+ for (int i = 0; i < l.getBatch(); i++) {
+ loss_sum += t[i];
+ }
+ loss = loss_sum / (float)l.getBatch();
+
+ if (opt.getWeightDecayType() == WeightDecayType::l2norm) {
+ loss += opt.getWeightDecayLambda() * 0.5 * (weight.l2norm());
+ }
+
+ } break;
+ case COST_UNKNOWN:
+ default:
+ break;
}
-
- } break;
- case COST_UNKNOWN:
- default:
- break;
}
- Tensor djdw = input.transpose().dot(djdb);
+ Tensor ret = djdb.dot(weight.transpose());
- ret = djdb.dot(weight.transpose());
+ Tensor djdw = input.transpose().dot(djdb);
opt.calculate(djdw, djdb, weight, bias, iteration, this->init_zero);
return ret;
}
-int BatchNormalizationLayer::initialize(int b, int h, int w, int id,
+int BatchNormalizationLayer::initialize(int b, int h, int w, bool last,
bool init_zero, WeightIniType wini) {
int status = ML_ERROR_NONE;
if (b <= 0 || h <= 0 || w <= 0) {
this->batch = b;
this->width = w;
this->height = h;
- this->index = id;
+ this->last_layer = last;
this->init_zero = init_zero;
this->gamma = Tensor(batch, w);
return this->opt.initialize(height, width, false);
}
-Tensor BatchNormalizationLayer::forwarding(Tensor in) {
+Tensor BatchNormalizationLayer::forwarding(Tensor in, int &status) {
Tensor temp;
hidden = in;
void BatchNormalizationLayer::copy(Layer *l) {
BatchNormalizationLayer *from = static_cast<BatchNormalizationLayer *>(l);
this->opt = from->opt;
- this->index = from->index;
+ this->last_layer = from->last_layer;
this->height = from->height;
this->width = from->width;
this->input.copy(from->input);
* @brief Layer Type String from configure file
* "input" : Input Layer Object
* "fully_conntected" : Fully Connected Layer Object
- * "output" : Output Layer Object
* "batch_normalization" : Batch Normalization Layer Object
* "unknown" : Batch Normalization Layer Object
*/
- std::array<std::string, 5> layer_string = {
- "input", "fully_connected", "output", "batch_normalization", "unknown"};
+ std::array<std::string, 4> layer_string = {"input", "fully_connected",
+ "batch_normalization", "unknown"};
/**
* @brief Weight Initialization Type String from configure file
int NeuralNetwork::init() {
int status = ML_ERROR_NONE;
- int id;
bool b_zero;
std::string l_type;
LayerType t;
break;
case LAYER_IN:
case LAYER_FC:
- case LAYER_OUT:
default:
h_size =
iniparser_getint(ini, (layers_name[i] + ":HiddenSize").c_str(), 0);
iniparser_getint(ini, "DataSet:BufferSize", batch_size));
for (unsigned int i = 0; i < layers_name.size(); i++) {
+ bool last = false;
l_type =
iniparser_getstring(ini, (layers_name[i] + ":Type").c_str(), unknown);
t = (LayerType)parseType(l_type, TOKEN_LAYER);
- id = iniparser_getint(ini, (layers_name[i] + ":Id").c_str(), 0);
+ if (i == (layers_name.size() - 1)) {
+ last = true;
+ }
b_zero =
iniparser_getboolean(ini, (layers_name[i] + ":Bias_zero").c_str(), true);
std::stringstream ss;
- ml_logi("%d : %s %d %d", id, l_type.c_str(), hidden_size[i], b_zero);
+ ml_logi("%d : %s %d %d", i, l_type.c_str(), hidden_size[i], b_zero);
switch (t) {
case LAYER_IN: {
InputLayer *input_layer = new (InputLayer);
input_layer->setType(t);
- status = input_layer->initialize(batch_size, 1, hidden_size[i], id,
+ status = input_layer->initialize(batch_size, 1, hidden_size[i], last,
b_zero, weight_ini);
NN_RETURN_STATUS();
case LAYER_FC: {
FullyConnectedLayer *fc_layer = new (FullyConnectedLayer);
fc_layer->setType(t);
+ status = fc_layer->setCost(cost);
+ NN_RETURN_STATUS();
if (i == 0) {
ml_loge("Error: Fully Connected Layer should be after "
"InputLayer.");
return ML_ERROR_INVALID_PARAMETER;
}
status = fc_layer->initialize(batch_size, hidden_size[i - 1],
- hidden_size[i], id, b_zero, weight_ini);
+ hidden_size[i], last, b_zero, weight_ini);
NN_RETURN_STATUS();
status = fc_layer->setOptimizer(opt);
NN_RETURN_STATUS();
NN_RETURN_STATUS();
layers.push_back(fc_layer);
} break;
- case LAYER_OUT: {
- OutputLayer *output_layer = new (OutputLayer);
- output_layer->setType(t);
- status = output_layer->setCost(cost);
- NN_RETURN_STATUS();
- if (i == 0) {
- ml_loge("Error: Output layer shouldn't be first layer of network");
- return ML_ERROR_INVALID_PARAMETER;
- }
- status = output_layer->initialize(batch_size, hidden_size[i - 1],
- hidden_size[i], id, b_zero, weight_ini);
- NN_RETURN_STATUS();
- status = output_layer->setOptimizer(opt);
- NN_RETURN_STATUS();
- status = output_layer->setActivation((ActiType)parseType(
- iniparser_getstring(ini, (layers_name[i] + ":Activation").c_str(),
- unknown),
- TOKEN_ACTI));
- NN_RETURN_STATUS();
- layers.push_back(output_layer);
- } break;
case LAYER_BN: {
BatchNormalizationLayer *bn_layer = new (BatchNormalizationLayer);
bn_layer->setType(t);
status = bn_layer->setOptimizer(opt);
NN_RETURN_STATUS();
- status = bn_layer->initialize(batch_size, 1, hidden_size[i], id, b_zero,
+ status = bn_layer->initialize(batch_size, 1, hidden_size[i], last, b_zero,
weight_ini);
NN_RETURN_STATUS();
layers.push_back(bn_layer);
/**
* @brief forward propagation using layers object which has layer
*/
-Tensor NeuralNetwork::forwarding(Tensor input) {
+Tensor NeuralNetwork::forwarding(Tensor input, int &status) {
Tensor X = input;
for (unsigned int i = 0; i < layers.size(); i++) {
- X = layers[i]->forwarding(X);
+ X = layers[i]->forwarding(X, status);
}
return X;
}
/**
* @brief forward propagation using layers object which has layer
*/
-Tensor NeuralNetwork::forwarding(Tensor input, Tensor output) {
+Tensor NeuralNetwork::forwarding(Tensor input, Tensor output, int &status) {
Tensor X = input;
Tensor Y2 = output;
for (unsigned int i = 0; i < layers.size(); i++) {
- X = layers[i]->forwarding(X, Y2);
+ X = layers[i]->forwarding(X, Y2, status);
}
return X;
}
* Call backwarding function of layer in reverse order
* No need to call at first Input Layer (No data to be updated)
*/
-void NeuralNetwork::backwarding(Tensor input, Tensor expected_output,
- int iteration) {
+int NeuralNetwork::backwarding(Tensor input, Tensor expected_output,
+ int iteration) {
+ int status = ML_ERROR_NONE;
Tensor Y2 = expected_output;
Tensor X = input;
- Tensor Y = forwarding(X);
+ Tensor Y = forwarding(X, status);
for (unsigned int i = layers.size() - 1; i > 0; i--) {
Y2 = layers[i]->backwarding(Y2, iteration);
}
+ return status;
}
float NeuralNetwork::getLoss() {
- OutputLayer *out = static_cast<OutputLayer *>((layers[layers.size() - 1]));
+ FullyConnectedLayer *out =
+ static_cast<FullyConnectedLayer *>((layers[layers.size() - 1]));
return out->getLoss();
}
for (int i = 0; i < batch_size; ++i) {
nntrainer::Tensor X = nntrainer::Tensor({in[i]});
nntrainer::Tensor Y2 = nntrainer::Tensor({label[i]});
- nntrainer::Tensor Y = forwarding(X, Y2);
+ nntrainer::Tensor Y = forwarding(X, Y2, status);
if (Y.argmax() == Y2.argmax())
right++;
valloss += getLoss();
"\n"
"Type = input"
"\n"
- "Id = 0"
- "\n"
"HiddenSize = 62720"
"\n"
"Bias_zero = true"
"\n"
"[outputlayer]"
"\n"
- "Type = output"
- "\n"
- "Id = 1"
+ "Type = fully_connected"
"\n"
"HiddenSize = 10"
"\n"
TEST(nntrainer_InputLayer, initialize_01_p) {
int status = ML_ERROR_NONE;
nntrainer::InputLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 1, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_NONE);
}
TEST(nntrainer_InputLayer, initialize_02_n) {
int status = ML_ERROR_NONE;
nntrainer::InputLayer layer;
- status = layer.initialize(1, 0, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 0, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
TEST(nntrainer_InputLayer, setOptimizer_01_p) {
int status = ML_ERROR_NONE;
nntrainer::InputLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 1, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
nntrainer::Optimizer op;
nntrainer::OptType t = nntrainer::OptType::adam;
nntrainer::OptParam p;
TEST(nntrainer_FullyConnectedLayer, initialize_01_p) {
int status = ML_ERROR_NONE;
nntrainer::FullyConnectedLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 1, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_NONE);
}
TEST(nntrainer_FullyConnectedLayer, initialize_02_n) {
int status = ML_ERROR_NONE;
nntrainer::FullyConnectedLayer layer;
- status = layer.initialize(1, 0, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 0, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
TEST(nntrainer_FullyConnectedLayer, initialize_03_n) {
int status = ML_ERROR_NONE;
nntrainer::FullyConnectedLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_UNKNOWN);
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Fully Connected Layer
- */
-TEST(nntrainer_FullyConnectedLayer, setOptimizer_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::FullyConnectedLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
- nntrainer::Optimizer op;
- nntrainer::OptType t = nntrainer::OptType::adam;
- nntrainer::OptParam p;
- status = op.setType(t);
- EXPECT_EQ(status, ML_ERROR_NONE);
- p.learning_rate = 0.001;
- p.beta1 = 0.9;
- p.beta2 = 0.9999;
- p.epsilon = 1e-7;
- p.weight_decay.type = nntrainer::WeightDecayType::l2norm;
- p.weight_decay.lambda = 0.001;
- status = op.setOptParam(p);
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = layer.setOptimizer(op);
+ status = layer.initialize(1, 1, 1, false, true, nntrainer::WEIGHT_UNKNOWN);
EXPECT_EQ(status, ML_ERROR_NONE);
}
/**
- * @brief Fully Connected Layer
+ * @brief FullyConnected Layer
*/
-TEST(nntrainer_FullyConnectedLayer, setActivation_01_p) {
+TEST(nntrainer_FullyConnectedLayer, initialize_04_p) {
int status = ML_ERROR_NONE;
nntrainer::FullyConnectedLayer layer;
- status = layer.setActivation(nntrainer::ACT_TANH);
+ status =
+ layer.initialize(1, 1, 1, true, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_NONE);
}
/**
- * @brief Fully Connected Layer
+ * @brief FullyConnected Layer
*/
-TEST(nntrainer_FullyConnectedLayer, setActivation_02_n) {
+TEST(nntrainer_FullyConnectedLayer, initialize_05_n) {
int status = ML_ERROR_NONE;
nntrainer::FullyConnectedLayer layer;
- status = layer.setActivation(nntrainer::ACT_UNKNOWN);
+ status =
+ layer.initialize(1, 0, 1, true, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
/**
- * @brief Output Layer
+ * @brief FullyConnected Layer
*/
-TEST(nntrainer_OutputLayer, initialize_01_p) {
+TEST(nntrainer_FullyConnectedLayer, initialize_06_n) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ nntrainer::FullyConnectedLayer layer;
+ status = layer.initialize(1, 1, 1, true, true, nntrainer::WEIGHT_UNKNOWN);
EXPECT_EQ(status, ML_ERROR_NONE);
}
/**
- * @brief Output Layer
- */
-TEST(nntrainer_OutputLayer, initialize_02_n) {
- int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
- status = layer.initialize(1, 0, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
- EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
-}
-
-/**
- * @brief Output Layer
+ * @brief Fully Connected Layer
*/
-TEST(nntrainer_OutputLayer, initialize_03_n) {
+TEST(nntrainer_FullyConnectedLayer, setOptimizer_01_p) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_UNKNOWN);
+ nntrainer::FullyConnectedLayer layer;
+ status =
+ layer.initialize(1, 1, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ nntrainer::Optimizer op;
+ nntrainer::OptType t = nntrainer::OptType::adam;
+ nntrainer::OptParam p;
+ status = op.setType(t);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+ p.learning_rate = 0.001;
+ p.beta1 = 0.9;
+ p.beta2 = 0.9999;
+ p.epsilon = 1e-7;
+ p.weight_decay.type = nntrainer::WeightDecayType::l2norm;
+ p.weight_decay.lambda = 0.001;
+ status = op.setOptParam(p);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+ status = layer.setOptimizer(op);
EXPECT_EQ(status, ML_ERROR_NONE);
}
/**
- * @brief Output Layer
+ * @brief FullyConnected Layer
*/
-TEST(nntrainer_OutputLayer, setOptimizer_01_p) {
+TEST(nntrainer_FullyConnectedLayer, setOptimizer_02_p) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ nntrainer::FullyConnectedLayer layer;
+ status =
+ layer.initialize(1, 1, 1, true, true, nntrainer::WEIGHT_XAVIER_NORMAL);
nntrainer::Optimizer op;
nntrainer::OptType t = nntrainer::OptType::adam;
nntrainer::OptParam p;
}
/**
- * @brief Output Layer
+ * @brief Fully Connected Layer
*/
-TEST(nntrainer_OutputLayer, setActivation_01_p) {
+TEST(nntrainer_FullyConnectedLayer, setActivation_01_p) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
- status = layer.setActivation(nntrainer::ACT_SIGMOID);
+ nntrainer::FullyConnectedLayer layer;
+ status = layer.setActivation(nntrainer::ACT_TANH);
EXPECT_EQ(status, ML_ERROR_NONE);
}
/**
- * @brief Output Layer
+ * @brief Fully Connected Layer
*/
-TEST(nntrainer_OutputLayer, setActivation_02_n) {
+TEST(nntrainer_FullyConnectedLayer, setActivation_02_n) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
+ nntrainer::FullyConnectedLayer layer;
status = layer.setActivation(nntrainer::ACT_UNKNOWN);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
/**
- * @brief Output Layer
+ * @brief FullyConnected Layer
*/
-TEST(nntrainer_OutputLayer, setCost_01_p) {
+TEST(nntrainer_FullyConnectedLayer, setCost_01_p) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
+ nntrainer::FullyConnectedLayer layer;
status = layer.setCost(nntrainer::COST_ENTROPY);
EXPECT_EQ(status, ML_ERROR_NONE);
}
/**
- * @brief Output Layer
+ * @brief FullyConnected Layer
*/
-TEST(nntrainer_OutputLayer, setCost_02_n) {
+TEST(nntrainer_FullyConnectedLayer, setCost_02_n) {
int status = ML_ERROR_NONE;
- nntrainer::OutputLayer layer;
+ nntrainer::FullyConnectedLayer layer;
status = layer.setCost(nntrainer::COST_UNKNOWN);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
TEST(nntrainer_BatchNormalizationLayer, initialize_01_p) {
int status = ML_ERROR_NONE;
nntrainer::BatchNormalizationLayer layer;
- status = layer.initialize(1, 1, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 1, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_NONE);
}
TEST(nntrainer_BatchNormalizationLayer, initialize_02_n) {
int status = ML_ERROR_NONE;
nntrainer::BatchNormalizationLayer layer;
- status = layer.initialize(1, 0, 1, 0, true, nntrainer::WEIGHT_XAVIER_NORMAL);
+ status =
+ layer.initialize(1, 0, 1, false, true, nntrainer::WEIGHT_XAVIER_NORMAL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
projects / platform / core / ml / nntrainer.git / commitdiff