--- /dev/null
+cmake_minimum_required(VERSION 2.8.3)
+
+project(nntrainer)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -g -pthread")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -g -std=c++11 -pthread")
+find_package(PkgConfig REQUIRED)
+
+pkg_check_modules(TFLITE tensorflow-lite)
+link_libraries(${TFLITE_LIBRARIES})
+
+set(INIPARSER ${PROJECT_SOURCE_DIR}/external/iniparser/src)
+
+include_directories( ${include_directories}
+ ${CMAKE_CURRENT_SOURCE_DIR}
+ ${TFLITE_INCLUDE_DIRS}
+ ${INIPARSER}
+ )
+
+set(SRCS
+ src/neuralnet.cpp
+ src/matrix.cpp
+ src/layers.cpp
+ ${INIPARSER}/iniparser.c
+ ${INIPARSER}/dictionary.c
+ )
+
+add_library( ${PROJECT_NAME} SHARED ${SRCS} )
--- /dev/null
+Subproject commit f858275f7f307eecba84c2f5429483f9f28007f8
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file layers.h
+ * @date 04 December 2019
+ * @brief This is Layer classes of Neural Network
+ * @see https://github.sec.samsung.net/jijoong-moon/Transfer-Learning.git
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+#ifndef __LAYERS_H__
+#define __LAYERS_H__
+
+#include <fstream>
+#include <iostream>
+#include <vector>
+#include "matrix.h"
+
+/**
+ * @Namespace Namespace of Layers
+ * @brief Namespace for Layers
+ */
+namespace Layers {
+
+/**
+ * @brief Enumeration of optimizer type
+ * 0. SGD ( Stocastic Gradient Descent )
+ * 1. ADAM ( Adaptive Moment Estimation )
+ * 2. Unknown
+ */
+typedef enum { OPT_SGD, OPT_ADAM, OPT_UNKNOWN } opt_type;
+
+/**
+ * @brief Enumeration of cost(loss) function type
+ * 0. MSR ( Mean Squared Roots )
+ * 1. ENTROPY ( Categorical Cross Entropy )
+ * 2. Unknown
+ */
+typedef enum { COST_MSR, COST_ENTROPY, COST_UNKNOWN } cost_type;
+
+/**
+ * @brief Enumeration of activation function type
+ * 0. tanh
+ * 1. sigmoid
+ * 2. Unknown
+ */
+typedef enum { ACT_TANH, ACT_SIGMOID, ACT_UNKNOWN } acti_type;
+
+/**
+ * @brief Enumeration of layer type
+ * 0. Input Layer type
+ * 1. Fully Connected Layer type
+ * 2. Output Layer type
+ * 3. Unknown
+ */
+typedef enum { LAYER_IN, LAYER_FC, LAYER_OUT, LAYER_UNKNOWN } layer_type;
+
+/**
+ * @brief type for the Optimizor to save hyper-parameter
+ */
+typedef struct {
+ opt_type type;
+ double learning_rate;
+ double beta1;
+ double beta2;
+ double epsilon;
+ acti_type activation;
+} Optimizer;
+
+/**
+ * @class Layer Base class for layers
+ * @brief Base class for all layers
+ */
+class Layer {
+ public:
+ /**
+ * @brief Destructor of Layer Class
+ */
+ virtual ~Layer(){};
+
+ /**
+ * @brief Forward Propation of neural Network
+ * @param[in] input Input Matrix taken by upper layer
+ * @retval Output Matrix
+ */
+ virtual Matrix forwarding(Matrix input) = 0;
+
+ /**
+ * @brief Back Propation of neural Network
+ * @param[in] input Input Matrix taken by lower layer
+ * @param[in] iteration Epoch value for the ADAM Optimizer
+ * @retval Output Matrix
+ */
+ virtual Matrix backwarding(Matrix input, int iteration) = 0;
+
+ /**
+ * @brief Initialize the layer
+ * - Weight(Height, Width), Bias(1, Width)
+ * @param[in] b batch
+ * @param[in] h Height
+ * @param[in] w Width
+ * @param[in] id index of this layer
+ * @param[in] init_zero Bias initialization with zero
+ */
+ virtual void initialize(int b, int h, int w, int id, bool init_zero) = 0;
+
+ /**
+ * @brief read layer Weight & Bias data from file
+ * @param[in] file input file stream
+ */
+ virtual void read(std::ifstream &file) = 0;
+
+ /**
+ * @brief save layer Weight & Bias data from file
+ * @param[in] file output file stream
+ */
+ virtual void save(std::ofstream &file) = 0;
+
+ /**
+ * @brief Optimizer Setter
+ * @param[in] opt Optimizer
+ */
+ virtual void setOptimizer(Optimizer opt) = 0;
+
+ /**
+ * @brief Layer type Setter
+ * @param[in] type layer type
+ */
+ void setType(layer_type type) { this->type = type; }
+
+ /**
+ * @brief Copy Layer
+ * @param[in] l Layer to be copied
+ */
+ virtual void copy(Layer *l) = 0;
+
+ /**
+ * @brief Input Matrix
+ */
+ Matrix Input;
+
+ /**
+ * @brief Hidden Layer Matrix which store the
+ * forwading result
+ */
+ Matrix hidden;
+
+ /**
+ * @brief Layer index
+ */
+ unsigned int index;
+
+ /**
+ * @brief batch size of Weight Data
+ */
+ unsigned int batch;
+
+ /**
+ * @brief width size of Weight Data
+ */
+ unsigned int width;
+
+ /**
+ * @brief height size of Weight Data
+ */
+ unsigned int height;
+
+ /**
+ * @brief Optimizer for this layer
+ */
+ Optimizer opt;
+
+ /**
+ * @brief Boolean for the Bias to set zero
+ */
+ bool init_zero;
+
+ /**
+ * @brief Layer type
+ */
+ layer_type type;
+
+ /**
+ * @brief Activation function pointer
+ */
+ double (*activation)(double);
+
+ /**
+ * @brief Activation Derivative function pointer
+ */
+ double (*activationPrime)(double);
+};
+
+/**
+ * @class Input Layer
+ * @brief Just Handle the Input of Network
+ */
+class InputLayer : public Layer {
+ public:
+ /**
+ * @brief Constructor of InputLayer
+ */
+ InputLayer(){};
+
+ /**
+ * @brief Destructor of InputLayer
+ */
+ ~InputLayer(){};
+
+ /**
+ * @brief No Weight data for this Input Layer
+ */
+ void read(std::ifstream &file){};
+
+ /**
+ * @brief No Weight data for this Input Layer
+ */
+ void save(std::ofstream &file){};
+
+ /**
+ * @brief It is back propagation of input layer.
+ * It return Input as it is.
+ * @param[in] input input Matrix from lower layer.
+ * @param[in] iteration Epoch Number for ADAM
+ * @retval
+ */
+ Matrix backwarding(Matrix input, int iteration) { return Input; };
+
+ /**
+ * @brief foward propagation : return Input Matrix
+ * It return Input as it is.
+ * @param[in] input input Matrix from lower layer.
+ * @retval return Input Matrix
+ */
+ Matrix forwarding(Matrix input);
+
+ /**
+ * @brief Set Optimizer
+ * @param[in] opt optimizer
+ */
+ void setOptimizer(Optimizer opt);
+
+ /**
+ * @brief Initializer of Input Layer
+ * @param[in] b batch size
+ * @param[in] h height
+ * @param[in] w width
+ * @param[in] id index of this layer
+ * @param[in] init_zero boolean to set Bias zero
+ */
+ void initialize(int b, int h, int w, int id, bool init_zero);
+
+ /**
+ * @brief Copy Layer
+ * @param[in] l layer to copy
+ */
+ void copy(Layer *l);
+};
+
+/**
+ * @class FullyConnecedLayer
+ * @brief fully connected layer
+ */
+class FullyConnectedLayer : public Layer {
+ public:
+ /**
+ * @brief Constructor of Fully Connected Layer
+ */
+ FullyConnectedLayer(){};
+
+ /**
+ * @brief Destructor of Fully Connected Layer
+ */
+ ~FullyConnectedLayer(){};
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file);
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file);
+
+ /**
+ * @brief forward propagation with input
+ * @param[in] input Input Matrix from upper layer
+ * @retval Activation(W x input + B)
+ */
+ Matrix forwarding(Matrix input);
+
+ /**
+ * @brief back propagation
+ * Calculate dJdB & dJdW & Update W & B
+ * @param[in] input Input Matrix from lower layer
+ * @param[in] iteration Number of Epoch for ADAM
+ * @retval dJdB x W Matrix
+ */
+ Matrix backwarding(Matrix input, int iteration);
+
+ /**
+ * @brief set optimizer
+ * @param[in] opt Optimizer
+ */
+ void setOptimizer(Optimizer opt);
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(Layer *l);
+
+ /**
+ * @brief initialize layer
+ * @param[in] b batch size
+ * @param[in] h height
+ * @param[in] w width
+ * @param[in] id layer index
+ * @param[in] init_zero boolean to set Bias zero
+ */
+ void initialize(int b, int h, int w, int id, bool init_zero);
+
+ private:
+ Matrix Weight;
+ Matrix Bias;
+
+ /**
+ * @brief First Momentum Matrix for the ADAM
+ */
+ Matrix M;
+
+ /**
+ * @brief Second Momentum Matrix for the ADAM
+ */
+ Matrix V;
+};
+
+/**
+ * @class OutputLayer
+ * @brief OutputLayer (has Cost Function & Weight, Bias)
+ */
+class OutputLayer : public Layer {
+ public:
+ /**
+ * @brief Constructor of OutputLayer
+ */
+ OutputLayer(){};
+
+ /**
+ * @brief Destructor of OutputLayer
+ */
+ ~OutputLayer(){};
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file);
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &flle);
+
+ /**
+ * @brief forward propagation with input
+ * @param[in] input Input Matrix from upper layer
+ * @retval Activation(W x input + B)
+ */
+ Matrix forwarding(Matrix input);
+
+ /**
+ * @brief back propagation
+ * Calculate dJdB & dJdW & Update W & B
+ * @param[in] input Input Matrix from lower layer
+ * @param[in] iteration Number of Epoch for ADAM
+ * @retval dJdB x W Matrix
+ */
+ Matrix backwarding(Matrix label, int iteration);
+
+ /**
+ * @brief set optimizer
+ * @param[in] opt Optimizer
+ */
+ void setOptimizer(Optimizer opt);
+
+ /**
+ * @brief initialize layer
+ * @param[in] b batch size
+ * @param[in] h height
+ * @param[in] w width
+ * @param[in] id layer index
+ * @param[in] init_zero boolean to set Bias zero
+ */
+ void initialize(int b, int w, int h, int id, bool init_zero);
+
+ /**
+ * @brief get Loss value
+ */
+ double getLoss() { return loss; }
+
+ /**
+ * @brief set cost function
+ * @param[in] c cost function type
+ */
+ void setCost(cost_type c) { this->cost = c; };
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(Layer *l);
+
+ private:
+ Matrix Weight;
+ Matrix Bias;
+ Matrix M;
+ Matrix V;
+ double loss;
+ cost_type cost;
+};
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file matrix.h
+ * @date 04 December 2019
+ * @brief This is Matrix class for calculation
+ * @see https://github.sec.samsung.net/jijoong-moon/Transfer-Learning.git
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __MATRIX_H__
+#define __MATRIX_H__
+
+#include <cmath>
+#include <fstream>
+#include <iostream>
+#include <vector>
+
+/**
+ * @class Matrix Class for Calculation
+ * @brief Matrix Class for Calculation
+ */
+class Matrix {
+ public:
+ /**
+ * @brief Constructor of Matrix
+ */
+ Matrix(){};
+
+ /**
+ * @brief Constructor of Matrix with batch size one
+ * @param[in] heihgt Height of Matrix
+ * @param[in] width Width of Matrix
+ */
+ Matrix(int height, int width);
+
+ /**
+ * @brief Constructor of Matrix
+ * @param[in] batch Batch of Matrix
+ * @param[in] heihgt Height of Matrix
+ * @param[in] width Width of Matrix
+ */
+ Matrix(int batch, int height, int width);
+
+ /**
+ * @brief Constructor of Matrix
+ * @param[in] data data for the Matrix with batch size one
+ */
+ Matrix(std::vector<std::vector<double>> const &data);
+
+ /**
+ * @brief Constructor of Matrix
+ * @param[in] data data for the Matrix
+ */
+ Matrix(std::vector<std::vector<std::vector<double>>> const &data);
+
+ /**
+ * @brief Multiply value element by element
+ * @param[in] value multiplier
+ * @retval Calculated Matrix
+ */
+ Matrix multiply(double const &value);
+
+ /**
+ * @brief Divide value element by element
+ * @param[in] value Divisor
+ * @retval Calculated Matrix
+ */
+ Matrix divide(double const &value);
+
+ /**
+ * @brief Add Matrix Element by Element
+ * @param[in] m Matrix to be added
+ * @retval Calculated Matrix
+ */
+ Matrix add(Matrix const &m) const;
+
+ /**
+ * @brief Add value Element by Element
+ * @param[in] value value to be added
+ * @retval Calculated Matrix
+ */
+ Matrix add(double const &value);
+
+ /**
+ * @brief Substract Matrix Element by Element
+ * @param[in] m Matrix to be added
+ * @retval Calculated Matrix
+ */
+ Matrix subtract(Matrix const &m) const;
+
+ /**
+ * @brief Multiply Matrix Element by Element ( Not the MxM )
+ * @param[in] m Matrix to be multiplied
+ * @retval Calculated Matrix
+ */
+ Matrix multiply(Matrix const &m) const;
+
+ /**
+ * @brief Divide Matrix Element by Element
+ * @param[in] m Divisor Matrix
+ * @retval Calculated Matrix
+ */
+ Matrix divide(Matrix const &m) const;
+
+ /**
+ * @brief Dot Product of Matrix ( equal MxM )
+ * @param[in] m Matrix
+ * @retval Calculated Matrix
+ */
+ Matrix dot(Matrix const &m) const;
+
+ /**
+ * @brief Transpose Matrix
+ * @retval Calculated Matrix
+ */
+ Matrix transpose() const;
+
+ /**
+ * @brief sum all the Matrix elements according to the batch
+ * @retval Calculated Matrix(batch, 1, 1)
+ */
+ Matrix sum() const;
+
+ /**
+ * @brief Averaging the Matrix elements according to the batch
+ * @retval Calculated Matrix(1, height, width)
+ */
+ Matrix average() const;
+
+ /**
+ * @brief Softmax the Matrix elements
+ * @retval Calculated Matrix
+ */
+ Matrix softmax() const;
+
+ /**
+ * @brief Fill the Matrix elements with zero
+ */
+ void setZero();
+
+ /**
+ * @brief Reduce Rank ( Matrix to Vector )
+ * @retval Saved vector
+ */
+ std::vector<double> Mat2Vec();
+
+ /**
+ * @brief Apply function element by element
+ * @param[in] *function function pointer applied
+ * @retval Matrix
+ */
+ Matrix applyFunction(double (*function)(double)) const;
+
+ /**
+ * @brief Print element
+ * @param[in] out out stream
+ * @retval Matrix
+ */
+ void print(std::ostream &out) const;
+
+ /**
+ * @brief Get Width of Matrix
+ * @retval int Width
+ */
+ int getWidth() { return width; };
+
+ /**
+ * @brief Get Height of Matrix
+ * @retval int Height
+ */
+ int getHeight() { return height; };
+
+ /**
+ * @brief Get Batch of Matrix
+ * @retval int Batch
+ */
+ int getBatch() { return batch; };
+
+ /**
+ * @brief Set the elelemnt value
+ * @param[in] batch batch location
+ * @param[in] i height location
+ * @param[in] j width location
+ * @param[in] value value to be stored
+ */
+ void setValue(int batch, int i, int j, double value);
+
+ /**
+ * @brief Copy the Matrix
+ * @param[in] from Matrix to be Copyed
+ * @retval Matix
+ */
+ Matrix ©(Matrix const &from);
+
+ /**
+ * @brief Save the Matrix into file
+ * @param[in] file output file stream
+ */
+ void save(std::ofstream &file);
+
+ /**
+ * @brief Read the Matrix from file
+ * @param[in] file input file stream
+ */
+ void read(std::ifstream &file);
+
+private:
+/**< handle the data as a std::vector type */
+ std::vector<std::vector<std::vector<double>>> data;
+ int height;
+ int width;
+ int batch;
+ int dim;
+};
+
+/**
+ * @brief Overriding output stream
+ */
+std::ostream &operator<<(std::ostream &out, Matrix const &m);
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file neuralnet.h
+ * @date 04 December 2019
+ * @brief This is Neural Network Class
+ * @see https://github.sec.samsung.net/jijoong-moon/Transfer-Learning.git
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+#ifndef __NEURALNET_H__
+#define __NEURALNET_H__
+
+#include <fstream>
+#include <iostream>
+#include <vector>
+#include "layers.h"
+#include "matrix.h"
+
+/**
+ * @Namespace Namespace of Network
+ * @brief Namespace for Network
+ */
+namespace Network {
+
+/**
+ * @brief Enumeration of Network Type
+ * 0. KNN ( k Nearest Neighbor )
+ * 1. REG ( Logistic Regression )
+ * 2. NEU ( Neural Network )
+ * 3. Unknown
+ */
+typedef enum { NET_KNN, NET_REG, NET_NEU, NET_UNKNOWN } net_type;
+
+/**
+ * @brief Enumeration for input configuration file parsing
+ * 0. OPT ( Optimizer Token )
+ * 1. COST ( Cost Function Token )
+ * 2. NET ( Network Token )
+ * 3. ACTI ( Activation Token )
+ * 4. LAYER ( Layer Token )
+ * 5. UNKNOWN
+ */
+typedef enum { TOKEN_OPT, TOKEN_COST, TOKEN_NET, TOKEN_ACTI, TOKEN_LAYER, TOKEN_UNKNOWN } input_type;
+
+/**
+ * @class NeuralNetwork Class
+ * @brief NeuralNetwork Class which has Network Configuration & Layers
+ */
+class NeuralNetwork {
+ public:
+ /**
+ * @brief Constructor of NeuralNetwork Class
+ */
+ NeuralNetwork(){};
+
+ /**
+ * @brief Constructor of NeuralNetwork Class with Configuration file path
+ */
+ NeuralNetwork(std::string config_path);
+
+ /**
+ * @brief Destructor of NeuralNetwork Class
+ */
+ ~NeuralNetwork(){};
+
+ /**
+ * @brief Get Loss
+ * @retval loss value
+ */
+ double getLoss();
+
+ /**
+ * @brief Set Loss
+ * @param[in] l loss value
+ */
+ void setLoss(double l);
+
+ /**
+ * @brief Initialize Network
+ */
+ void init();
+
+ /**
+ * @brief forward propagation
+ * @param[in] input Input Matrix X
+ * @retval Output Matrix Y
+ */
+ Matrix forwarding(Matrix input);
+
+ /**
+ * @brief back propagation to update W & B
+ * @param[in] input Input Matrix X
+ * @param[in] expectedOutput Lable Matrix Y
+ * @param[in] iteration Epoch Number for ADAM
+ */
+ void backwarding(Matrix input, Matrix expectedOutput, int iteration);
+
+ /**
+ * @brief save W & B into file
+ */
+ void saveModel();
+
+ /**
+ * @brief read W & B from file
+ */
+ void readModel();
+
+ /**
+ * @brief set configuration file
+ * @param[in] config_path configuration file path
+ */
+ void setConfig(std::string config_path);
+
+ /**
+ * @brief get Epoch
+ * @retval epoch
+ */
+ unsigned int getEpoch() { return epoch; };
+
+ /**
+ * @brief Copy Neural Network
+ * @param[in] from NeuralNetwork Object to copy
+ * @retval NeuralNewtork Object copyed
+ */
+ NeuralNetwork ©(NeuralNetwork &from);
+
+ /**
+ * @brief finalize NeuralNetwork Object
+ */
+ void finalize();
+
+ private:
+ /**
+ * @brief batch size
+ */
+ int batchsize;
+
+ /**
+ * @brief function pointer for activation
+ */
+ double (*activation)(double);
+
+ /**
+ * @brief function pointer for derivative of activation
+ */
+ double (*activationPrime)(double);
+
+ /**
+ * @brief learning rate
+ */
+ double learning_rate;
+
+ /**
+ * @brief Maximum Epoch
+ */
+ unsigned int epoch;
+
+ /**
+ * @brief loss
+ */
+ double loss;
+
+ /**
+ * @brief boolean to set the Bias zero
+ */
+ bool init_zero;
+
+ /**
+ * @brief Cost Function type
+ */
+ Layers::cost_type cost;
+
+ /**
+ * @brief Model path to save or read
+ */
+ std::string model;
+
+ /**
+ * @brief Configuration file path
+ */
+ std::string config;
+
+ /**
+ * @brief Optimizer
+ */
+ Layers::Optimizer opt;
+
+ /**
+ * @brief Network Type
+ */
+ net_type nettype;
+
+ /**
+ * @brief vector for store layer pointers.
+ */
+ std::vector<Layers::Layer *> layers;
+};
+}
+
+#endif
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file layers.cpp
+ * @date 04 December 2019
+ * @brief This is Layers Classes for Neural Network
+ * @see https://github.sec.samsung.net/jijoong-moon/Transfer-Learning.git
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include "include/layers.h"
+#include <assert.h>
+
+/**
+ * @brief random function
+ * @param[in] x double
+ */
+double random(double x) { return (double)(rand() % 10000 + 1) / 10000 - 0.5; }
+
+/**
+ * @brief sigmoid activation function
+ * @param[in] x input
+ */
+
+double sigmoid(double x) { return 1 / (1 + exp(-x)); }
+
+/**
+ * @brief derivative sigmoid function
+ * @param[in] x input
+ */
+double sigmoidePrime(double x) { return exp(-x) / (pow(1 + exp(-x), 2)); }
+
+/**
+ * @brief derivative tanh function
+ * @param[in] x input
+ */
+double tanhPrime(double x) {
+ double th = tanh(x);
+ return 1.0 - th * th;
+}
+
+namespace Layers {
+
+void InputLayer::setOptimizer(Optimizer opt) {
+ this->opt = opt;
+ switch (opt.activation) {
+ case ACT_TANH:
+ activation = tanh;
+ activationPrime = tanhPrime;
+ break;
+ case ACT_SIGMOID:
+ activation = sigmoid;
+ activationPrime = sigmoidePrime;
+ break;
+ default:
+ break;
+ }
+}
+
+void InputLayer::copy(Layer *l) {
+ InputLayer *from = static_cast<InputLayer *>(l);
+ this->opt = from->opt;
+ this->index = from->index;
+ this->height = from->height;
+ this->width = from->width;
+ this->Input.copy(from->Input);
+ this->hidden.copy(from->hidden);
+}
+
+Matrix InputLayer::forwarding(Matrix input) {
+ Input = input;
+ return Input;
+}
+
+void InputLayer::initialize(int b, int h, int w, int id, bool init_zero) {
+ batch = b;
+ width = w;
+ height = h;
+ index = 0;
+}
+
+void FullyConnectedLayer::initialize(int b, int h, int w, int id, bool init_zero) {
+ this->batch = b;
+ this->width = w;
+ this->height = h;
+ this->index = id;
+ this->init_zero = init_zero;
+
+ Weight = Matrix(h, w);
+ Bias = Matrix(1, w);
+
+ Weight = Weight.applyFunction(random);
+ if (init_zero) {
+ Bias.setZero();
+ } else {
+ Bias = Bias.applyFunction(random);
+ }
+}
+
+void FullyConnectedLayer::setOptimizer(Optimizer opt) {
+ this->opt = opt;
+ switch (opt.activation) {
+ case ACT_TANH:
+ activation = tanh;
+ activationPrime = tanhPrime;
+ break;
+ case ACT_SIGMOID:
+ activation = sigmoid;
+ activationPrime = sigmoidePrime;
+ break;
+ default:
+ break;
+ }
+ if (opt.type == OPT_ADAM) {
+ M = Matrix(height, width);
+ V = Matrix(height, width);
+ M.setZero();
+ V.setZero();
+ }
+}
+
+Matrix FullyConnectedLayer::forwarding(Matrix input) {
+ Input = input;
+ hidden = Input.dot(Weight).add(Bias).applyFunction(activation);
+ return hidden;
+}
+
+void FullyConnectedLayer::read(std::ifstream &file) {
+ Weight.read(file);
+ Bias.read(file);
+}
+
+void FullyConnectedLayer::save(std::ofstream &file) {
+ Weight.save(file);
+ Bias.save(file);
+}
+
+void FullyConnectedLayer::copy(Layer *l) {
+ FullyConnectedLayer *from = static_cast<FullyConnectedLayer *>(l);
+ this->opt = from->opt;
+ this->index = from->index;
+ this->height = from->height;
+ this->width = from->width;
+ this->Input.copy(from->Input);
+ this->hidden.copy(from->hidden);
+ this->Weight.copy(from->Weight);
+ this->Bias.copy(from->Bias);
+}
+
+Matrix FullyConnectedLayer::backwarding(Matrix derivative, int iteration) {
+ Matrix dJdB = derivative.multiply(Input.dot(Weight).add(Bias).applyFunction(activationPrime));
+ Matrix dJdW = Input.transpose().dot(dJdB);
+ Matrix ret = dJdB.dot(Weight.transpose());
+
+ switch (opt.type) {
+ case OPT_SGD:
+ Weight = Weight.subtract(dJdW.average().multiply(opt.learning_rate));
+ break;
+ case OPT_ADAM:
+ M = M.multiply(opt.beta1).add(dJdW.average().multiply(1 - opt.beta1));
+ V = V.multiply(opt.beta2).add((dJdW.average().multiply(dJdW.average())).multiply(1 - opt.beta2));
+ M.divide(1 - pow(opt.beta1, iteration + 1));
+ V.divide(1 - pow(opt.beta2, iteration + 1));
+ Weight = Weight.subtract((M.divide(V.applyFunction(sqrt).add(opt.epsilon))).multiply(opt.learning_rate));
+ break;
+ default:
+ break;
+ }
+
+ if (!this->init_zero) {
+ Bias = Bias.subtract(dJdB.average().multiply(opt.learning_rate));
+ }
+
+ return ret;
+}
+
+void OutputLayer::initialize(int b, int h, int w, int id, bool init_zero) {
+ this->batch = b;
+ this->width = w;
+ this->height = h;
+ this->index = id;
+ this->init_zero = init_zero;
+ Weight = Matrix(h, w);
+ Bias = Matrix(1, w);
+ this->cost = cost;
+
+ Weight = Weight.applyFunction(random);
+ if (init_zero) {
+ Bias.setZero();
+ } else {
+ Bias = Bias.applyFunction(random);
+ }
+}
+
+Matrix OutputLayer::forwarding(Matrix input) {
+ Input = input;
+ if (cost == COST_ENTROPY)
+ hidden = input.dot(Weight).applyFunction(activation);
+ else
+ hidden = input.dot(Weight).add(Bias).applyFunction(activation);
+ return hidden;
+}
+
+void OutputLayer::read(std::ifstream &file) {
+ Weight.read(file);
+ Bias.read(file);
+}
+
+void OutputLayer::save(std::ofstream &file) {
+ Weight.save(file);
+ Bias.save(file);
+}
+
+void OutputLayer::copy(Layer *l) {
+ OutputLayer *from = static_cast<OutputLayer *>(l);
+ this->opt = from->opt;
+ this->index = from->index;
+ this->height = from->height;
+ this->width = from->width;
+ this->Input.copy(from->Input);
+ this->hidden.copy(from->hidden);
+ this->Weight.copy(from->Weight);
+ this->Bias.copy(from->Bias);
+ this->loss = from->loss;
+}
+
+void OutputLayer::setOptimizer(Optimizer opt) {
+ this->opt = opt;
+ switch (opt.activation) {
+ case ACT_TANH:
+ activation = tanh;
+ activationPrime = tanhPrime;
+ break;
+ case ACT_SIGMOID:
+ activation = sigmoid;
+ activationPrime = sigmoidePrime;
+ break;
+ default:
+ break;
+ }
+
+ if (opt.type == OPT_ADAM) {
+ M = Matrix(height, width);
+ V = Matrix(height, width);
+ M.setZero();
+ V.setZero();
+ }
+}
+
+Matrix OutputLayer::backwarding(Matrix label, int iteration) {
+ double lossSum = 0.0;
+ Matrix Y2 = label;
+ Matrix Y = hidden;
+ Matrix ret;
+ Matrix dJdB;
+
+ if (cost == COST_ENTROPY) {
+ dJdB = Y.subtract(Y2);
+ Matrix temp = ((Y2.multiply(-1.0).transpose().dot(Y.add(opt.epsilon).applyFunction(log)))
+ .subtract(Y2.multiply(-1.0).add(1.0).transpose().dot(
+ Y.multiply(-1.0).add(1.0).add(opt.epsilon).applyFunction(log))));
+ loss = (1.0 / Y.Mat2Vec().size()) * temp.Mat2Vec()[0];
+ } else {
+ Matrix sub = Y2.subtract(Y);
+ Matrix l = (sub.multiply(sub)).sum().multiply(0.5);
+ std::vector<double> t = l.Mat2Vec();
+ for (int i = 0; i < l.getBatch(); i++) {
+ lossSum += t[i];
+ }
+
+ loss = lossSum / (double)l.getBatch();
+
+ dJdB = Y.subtract(Y2).multiply(Input.dot(Weight).add(Bias).applyFunction(activationPrime));
+ }
+
+ Matrix dJdW = Input.transpose().dot(dJdB);
+ ret = dJdB.dot(Weight.transpose());
+
+ switch (opt.type) {
+ case Layers::OPT_SGD:
+ Weight = Weight.subtract(dJdW.average().multiply(opt.learning_rate));
+ break;
+ case Layers::OPT_ADAM:
+ M = M.multiply(opt.beta1).add(dJdW.average().multiply(1 - opt.beta1));
+ V = V.multiply(opt.beta2).add((dJdW.average().multiply(dJdW.average())).multiply(1 - opt.beta2));
+ M.divide(1 - pow(opt.beta1, iteration + 1));
+ V.divide(1 - pow(opt.beta2, iteration + 1));
+ Weight = Weight.subtract((M.divide(V.applyFunction(sqrt).add(opt.epsilon))).multiply(opt.learning_rate));
+ break;
+ default:
+ break;
+ }
+
+ if (!this->init_zero) {
+ Bias = Bias.subtract(dJdB.average().multiply(opt.learning_rate));
+ }
+
+ return ret;
+}
+}
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file matrix.cpp
+ * @date 04 December 2019
+ * @brief This is Matrix class for calculation
+ * @see https://github.sec.samsung.net/jijoong-moon/Transfer-Learning.git
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include "include/matrix.h"
+#include <assert.h>
+#include <stdio.h>
+#include <sstream>
+
+Matrix::Matrix(int height, int width) {
+ this->height = height;
+ this->width = width;
+ this->batch = 1;
+ this->dim = 2;
+ this->data.push_back(std::vector<std::vector<double>>(height, std::vector<double>(width)));
+}
+
+Matrix::Matrix(int batch, int height, int width) {
+ this->height = height;
+ this->width = width;
+ this->batch = batch;
+ this->dim = 3;
+
+ for (int i = 0; i < batch; i++) {
+ this->data.push_back(std::vector<std::vector<double>>(height, std::vector<double>(width)));
+ }
+}
+
+Matrix::Matrix(std::vector<std::vector<double>> const &data) {
+ assert(data.size() != 0);
+ this->height = data.size();
+ this->width = data[0].size();
+ this->batch = 1;
+ this->dim = 2;
+ this->data.push_back(data);
+}
+
+Matrix::Matrix(std::vector<std::vector<std::vector<double>>> const &data) {
+ assert(data.size() != 0 && data[0].size() != 0);
+ this->batch = data.size();
+ this->height = data[0].size();
+ this->width = data[0][0].size();
+ this->dim = 3;
+ this->data = data;
+}
+
+Matrix Matrix::multiply(double const &value) {
+ Matrix result(batch, height, width);
+ int i, j, k;
+
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] * value;
+ }
+ }
+ }
+
+ return result;
+}
+
+Matrix Matrix::divide(double const &value) {
+ Matrix result(batch, height, width);
+ int i, j, k;
+
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] / value;
+ }
+ }
+ }
+
+ return result;
+}
+
+Matrix Matrix::add(double const &value) {
+ Matrix result(batch, height, width);
+ int i, j, k;
+
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] + value;
+ }
+ }
+ }
+
+ return result;
+}
+
+Matrix Matrix::add(Matrix const &m) const {
+ assert(height == m.height && width == m.width);
+
+ Matrix result(batch, height, width);
+ int i, j, k;
+ if (m.batch == 1) {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] + m.data[0][i][j];
+ }
+ }
+ }
+ } else {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] + m.data[k][i][j];
+ }
+ }
+ }
+ }
+ return result;
+}
+
+Matrix Matrix::subtract(Matrix const &m) const {
+ assert(height == m.height && width == m.width);
+ Matrix result(batch, height, width);
+ int i, j, k;
+
+ if (m.batch == 1) {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] - m.data[0][i][j];
+ }
+ }
+ }
+ } else {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] - m.data[k][i][j];
+ }
+ }
+ }
+ }
+ return result;
+}
+
+Matrix Matrix::multiply(Matrix const &m) const {
+ assert(height == m.height && width == m.width);
+ Matrix result(batch, height, width);
+
+ int i, j, k;
+
+ if (m.batch == 1) {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] * m.data[0][i][j];
+ }
+ }
+ }
+ } else {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] * m.data[k][i][j];
+ }
+ }
+ }
+ }
+
+ return result;
+}
+
+Matrix Matrix::divide(Matrix const &m) const {
+ assert(height == m.height && width == m.width);
+ Matrix result(batch, height, width);
+
+ int i, j, k;
+
+ if (m.batch == 1) {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] / m.data[0][i][j];
+ }
+ }
+ }
+ } else {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = data[k][i][j] / m.data[k][i][j];
+ }
+ }
+ }
+ }
+
+ return result;
+}
+
+/**
+ * This is to sum the Matrix data according to the batch.
+ * Therefore the result has M(batch, 1, 1) dimension.
+ */
+Matrix Matrix::sum() const {
+ int i, j, k;
+ Matrix ret(batch, 1, 1);
+
+ for (k = 0; k < batch; k++) {
+ ret.data[k][0][0] = 0.0;
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ ret.data[k][0][0] += data[k][i][j];
+ }
+ }
+ }
+
+ return ret;
+}
+
+/**
+ * If the batch sizeo of m is one, the it is reused for
+ * every calculation along with batch
+ */
+Matrix Matrix::dot(Matrix const &m) const {
+ assert(width == m.height);
+ int i, j, h, k, mwidth = m.width;
+ double w = 0;
+
+ Matrix result(batch, height, mwidth);
+ if (m.batch == 1) {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < mwidth; j++) {
+ for (h = 0; h < width; h++) {
+ w += data[k][i][h] * m.data[0][h][j];
+ }
+ result.data[k][i][j] = w;
+ w = 0;
+ }
+ }
+ }
+ } else {
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < mwidth; j++) {
+ for (h = 0; h < width; h++) {
+ w += data[k][i][h] * m.data[k][h][j];
+ }
+ result.data[k][i][j] = w;
+ w = 0;
+ }
+ }
+ }
+ }
+
+ return result;
+}
+
+Matrix Matrix::transpose() const {
+ Matrix result(batch, width, height);
+ int i, j, k;
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < width; i++) {
+ for (j = 0; j < height; j++) {
+ result.data[k][i][j] = data[k][j][i];
+ }
+ }
+ }
+ return result;
+}
+
+Matrix Matrix::applyFunction(double (*function)(double)) const {
+ Matrix result(batch, height, width);
+ int i, j, k;
+
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ result.data[k][i][j] = (*function)(data[k][i][j]);
+ }
+ }
+ }
+ return result;
+}
+
+void Matrix::print(std::ostream &out) const {
+ int i, j, k;
+ std::stringstream ss;
+ for (k = 0; k < batch; k++) {
+ for (i = 0; i < height; i++) {
+ for (j = 0; j < width; j++) {
+ out << data[k][i][j] << " ";
+ }
+ out << std::endl;
+ }
+ out << std::endl;
+ }
+}
+
+std::ostream &operator<<(std::ostream &out, Matrix const &m) {
+ m.print(out);
+ return out;
+}
+
+Matrix &Matrix::copy(const Matrix &from) {
+ if (this != &from && from.data.size() != 0) {
+ height = from.height;
+ width = from.width;
+ batch = from.batch;
+ for (int k = 0; k < batch; k++) {
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ data[k][i][j] = from.data[k][i][j];
+ }
+ }
+ }
+ }
+ return *this;
+}
+
+/**
+ * This generate one dimension vector has the every element in Matrix
+ */
+std::vector<double> Matrix::Mat2Vec() {
+ std::vector<double> ret;
+ for (int k = 0; k < batch; k++)
+ for (int i = 0; i < height; i++)
+ for (int j = 0; j < width; j++)
+ ret.push_back(data[k][i][j]);
+
+ return ret;
+}
+
+void Matrix::save(std::ofstream &file) {
+ for (int k = 0; k < batch; k++) {
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ file.write((char *)&data[k][i][j], sizeof(double));
+ }
+ }
+ }
+}
+
+void Matrix::read(std::ifstream &file) {
+ for (int k = 0; k < batch; k++) {
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ file.read((char *)&data[k][i][j], sizeof(double));
+ }
+ }
+ }
+}
+
+/**
+ * This calculates average value according to the batch direction.
+ * That is the why it has (1, height, width) dimension.
+ */
+Matrix Matrix::average() const {
+ if (batch == 1)
+ return *this;
+
+ Matrix result(1, height, width);
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ result.data[0][i][j] = 0.0;
+ for (int k = 0; k < batch; k++) {
+ result.data[0][i][j] += data[k][i][j];
+ }
+ result.data[0][i][j] = result.data[0][i][j] / (double)batch;
+ }
+ }
+ return result;
+}
+
+void Matrix::setZero() {
+ for (int k = 0; k < batch; k++) {
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ this->data[k][i][j] = 0.0;
+ }
+ }
+ }
+}
+
+Matrix Matrix::softmax() const {
+ Matrix result(batch, height, width);
+ Matrix divisor(batch, height, 1);
+
+ divisor.setZero();
+
+ for (int k = 0; k < batch; k++) {
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ divisor.data[k][i][0] += exp(this->data[k][i][j]);
+ }
+ }
+ }
+
+ for (int k = 0; k < batch; k++) {
+ for (int i = 0; i < height; i++) {
+ for (int j = 0; j < width; j++) {
+ result.data[k][i][j] = exp(this->data[k][i][j]) / divisor.data[k][i][0];
+ }
+ }
+ }
+ return result;
+}
+
+void Matrix::setValue(int batch, int height, int width, double value) { this->data[batch][height][width] = value; }
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file neuralnet.cpp
+ * @date 04 December 2019
+ * @brief This is Neural Network Class
+ * @see https://github.sec.samsung.net/jijoong-moon/Transfer-Learning.git
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include "include/neuralnet.h"
+#include <assert.h>
+#include <stdio.h>
+#include <cmath>
+#include <sstream>
+#include "iniparser.h"
+
+/**
+ * @brief compare character to remove case sensitivity
+ * @param[in] c1 characer #1
+ * @param[in] c2 characer #2
+ * @retval boolean true if they are same
+ */
+bool compareChar(char &c1, char &c2) {
+ if (c1 == c2)
+ return true;
+ else if (std::toupper(c1) == std::toupper(c2))
+ return true;
+ return false;
+}
+
+/**
+ * @brief compare string with case insensitive
+ * @param[in] str1 string #1
+ * @param[in] str2 string #2
+ * @retval boolean true if they are same
+ */
+bool caseInSensitiveCompare(std::string &str1, std::string &str2) {
+ return ((str1.size() == str2.size()) && std::equal(str1.begin(), str1.end(), str2.begin(), &compareChar));
+}
+
+namespace Network {
+
+/**
+ * @brief Optimizer String from configure file
+ * "sgd" : Stochestic Gradient Descent
+ * "adam" : Adaptive Moment Estimation
+ */
+std::vector<std::string> Optimizer_string = {"sgd", "adam"};
+
+/**
+ * @brief Cost Function String from configure file
+ * "msr" : Mean Squared Roots
+ * "caterogical" : Categorical Cross Entropy
+ */
+std::vector<std::string> Cost_string = {"msr", "categorical"};
+
+/**
+ * @brief Network Type String from configure file
+ * "knn" : K Neearest Neighbor
+ * "regression" : Logistic Regression
+ * "neuralnet" : Neural Network
+ */
+std::vector<std::string> NetworkType_string = {"knn", "regression", "neuralnet"};
+
+/**
+ * @brief Activation Type String from configure file
+ * "tanh" : tanh
+ * "sigmoid" : sigmoid
+ */
+std::vector<std::string> activation_string = {"tanh", "sigmoid"};
+
+/**
+ * @brief Layer Type String from configure file
+ * "InputLayer" : InputLayer Object
+ * "FullyConnectedLayer" : Fully Connected Layer Object
+ * "OutputLayer" : Output Layer Object
+ */
+std::vector<std::string> layer_string = {"InputLayer", "FullyConnectedLayer", "OutputLayer"};
+
+/**
+ * @brief Check Existance of File
+ * @param[in] filename file path to check
+ * @retval boolean true if exists
+ */
+static bool is_file_exist(std::string filename) {
+ std::ifstream infile(filename);
+ return infile.good();
+}
+
+/**
+ * @brief Parsing Layer Name
+ * @param[in] string layer name
+ * @retval vector stored layer name
+ */
+std::vector<std::string> parseLayerName(std::string ll) {
+ std::vector<std::string> ret;
+ std::istringstream ss(ll);
+ do {
+ std::string word;
+ ss >> word;
+ if (word.compare("") != 0)
+ ret.push_back(word);
+ } while (ss);
+
+ return ret;
+}
+
+/**
+ * @brief Parsing Configuration Token
+ * @param[in] ll string to be parsed
+ * @param[in] t Token type
+ * @retval int enumerated type
+ */
+unsigned int parseType(std::string ll, input_type t) {
+ int ret;
+ unsigned int i;
+
+ switch (t) {
+ case TOKEN_OPT:
+ for (i = 0; i < Optimizer_string.size(); i++) {
+ if (caseInSensitiveCompare(Optimizer_string[i], ll)) {
+ return (i);
+ }
+ }
+ ret = i - 1;
+ break;
+ case TOKEN_COST:
+ for (i = 0; i < Cost_string.size(); i++) {
+ if (caseInSensitiveCompare(Cost_string[i], ll)) {
+ return (i);
+ }
+ }
+ ret = i - 1;
+ break;
+ case TOKEN_NET:
+ for (i = 0; i < NetworkType_string.size(); i++) {
+ if (caseInSensitiveCompare(NetworkType_string[i], ll)) {
+ return (i);
+ }
+ }
+ ret = i - 1;
+ break;
+ case TOKEN_ACTI:
+ for (i = 0; i < activation_string.size(); i++) {
+ if (caseInSensitiveCompare(activation_string[i], ll)) {
+ return (i);
+ }
+ }
+ ret = i - 1;
+ break;
+ case TOKEN_LAYER:
+ for (i = 0; i < layer_string.size(); i++) {
+ if (caseInSensitiveCompare(layer_string[i], ll)) {
+ return (i);
+ }
+ }
+ ret = i - 1;
+ break;
+ case TOKEN_UNKNOWN:
+ default:
+ ret = 3;
+ break;
+ }
+ return ret;
+}
+
+NeuralNetwork::NeuralNetwork(std::string config) { this->config = config; }
+
+void NeuralNetwork::setConfig(std::string config) { this->config = config; }
+
+void NeuralNetwork::init() {
+ int w, h, id;
+ bool b_zero;
+ std::string l_type;
+ Layers::layer_type t;
+ std::string inifile = config;
+ dictionary *ini = iniparser_load(inifile.c_str());
+
+ if (ini == NULL) {
+ fprintf(stderr, "cannot parse file: %s\n", inifile.c_str());
+ }
+
+ nettype = (Network::net_type)parseType(iniparser_getstring(ini, "Network:Type", NULL), TOKEN_NET);
+ std::vector<std::string> layers_name = parseLayerName(iniparser_getstring(ini, "Network:Layers", NULL));
+ learning_rate = iniparser_getdouble(ini, "Network:Learning_rate", 0.0);
+ opt.learning_rate = learning_rate;
+ epoch = iniparser_getint(ini, "Network:Epoch", 100);
+ opt.type = (Layers::opt_type)parseType(iniparser_getstring(ini, "Network:Optimizer", NULL), TOKEN_OPT);
+ opt.activation = (Layers::acti_type)parseType(iniparser_getstring(ini, "Network:Activation", NULL), TOKEN_ACTI);
+ cost = (Layers::cost_type)parseType(iniparser_getstring(ini, "Network:Cost", NULL), TOKEN_COST);
+
+ model = iniparser_getstring(ini, "Network:Model", "model.bin");
+ batchsize = iniparser_getint(ini, "Network:minibatch", 1);
+
+ opt.beta1 = iniparser_getdouble(ini, "Network:beta1", 0.0);
+ opt.beta2 = iniparser_getdouble(ini, "Network:beta2", 0.0);
+ opt.epsilon = iniparser_getdouble(ini, "Network:epsilon", 0.0);
+
+ for (unsigned int i = 0; i < layers_name.size(); i++)
+ std::cout << layers_name[i] << std::endl;
+
+ loss = 100000.0;
+
+ for (unsigned int i = 0; i < layers_name.size(); i++) {
+ l_type = iniparser_getstring(ini, (layers_name[i] + ":Type").c_str(), NULL);
+ t = (Layers::layer_type)parseType(l_type, TOKEN_LAYER);
+ w = iniparser_getint(ini, (layers_name[i] + ":Width").c_str(), 1);
+ h = iniparser_getint(ini, (layers_name[i] + ":Height").c_str(), 1);
+ id = iniparser_getint(ini, (layers_name[i] + ":Id").c_str(), 0);
+ b_zero = iniparser_getboolean(ini, (layers_name[i] + ":Bias_zero").c_str(), true);
+ std::cout << l_type << " " << t << " " << w << " " << h << " " << b_zero << " " << id << std::endl;
+ switch (t) {
+ case Layers::LAYER_IN: {
+ Layers::InputLayer *inputlayer = new (Layers::InputLayer);
+ inputlayer->setType(t);
+ inputlayer->initialize(batchsize, h, w, id, b_zero);
+ inputlayer->setOptimizer(opt);
+ layers.push_back(inputlayer);
+ } break;
+ case Layers::LAYER_FC: {
+ Layers::FullyConnectedLayer *fclayer = new (Layers::FullyConnectedLayer);
+ fclayer->setType(t);
+ fclayer->initialize(batchsize, h, w, id, b_zero);
+ fclayer->setOptimizer(opt);
+ layers.push_back(fclayer);
+ } break;
+ case Layers::LAYER_OUT: {
+ Layers::OutputLayer *outputlayer = new (Layers::OutputLayer);
+ outputlayer->setType(t);
+ outputlayer->initialize(batchsize, h, w, id, b_zero);
+ outputlayer->setOptimizer(opt);
+ outputlayer->setCost(cost);
+ layers.push_back(outputlayer);
+ } break;
+ case Layers::LAYER_UNKNOWN:
+ break;
+ default:
+ break;
+ }
+ }
+
+ iniparser_freedict(ini);
+}
+
+/**
+ * @brief free layers
+ */
+void NeuralNetwork::finalize() {
+ for (unsigned int i = 0; i < layers.size(); i++) {
+ delete layers[i];
+ }
+}
+
+/**
+ * @brief forward propagation using layers object which has layer
+ */
+Matrix NeuralNetwork::forwarding(Matrix input) {
+ Matrix X = input;
+ for (unsigned int i = 0; i < layers.size(); i++) {
+ X = layers[i]->forwarding(X);
+ }
+ return X;
+}
+
+/**
+ * @brief back propagation
+ * Call backwarding function of layer in reverse order
+ * No need to call at first Input Layer (No data to be updated)
+ */
+void NeuralNetwork::backwarding(Matrix input, Matrix expected_output, int iteration) {
+ Matrix Y2 = expected_output;
+ Matrix X = input;
+ Matrix Y = forwarding(X);
+
+ for (unsigned int i = layers.size() - 1; i > 0; i--) {
+ Y2 = layers[i]->backwarding(Y2, i);
+ }
+}
+
+double NeuralNetwork::getLoss() {
+ Layers::OutputLayer *out = static_cast<Layers::OutputLayer *>((layers[layers.size() - 1]));
+ return out->getLoss();
+}
+
+void NeuralNetwork::setLoss(double l) { loss = l; }
+
+NeuralNetwork &NeuralNetwork::copy(NeuralNetwork &from) {
+ if (this != &from) {
+ batchsize = from.batchsize;
+ learning_rate = from.learning_rate;
+ loss = from.loss;
+ opt = from.opt;
+
+ for (unsigned int i = 0; i < layers.size(); i++)
+ layers[i]->copy(from.layers[i]);
+ }
+ return *this;
+}
+
+/**
+ * @brief save model
+ * save Weight & Bias Data into file by calling save from layer
+ */
+void NeuralNetwork::saveModel() {
+ std::ofstream modelFile(model, std::ios::out | std::ios::binary);
+ for (unsigned int i = 0; i < layers.size(); i++)
+ layers[i]->save(modelFile);
+ modelFile.close();
+}
+
+/**
+ * @brief read model
+ * read Weight & Bias Data into file by calling save from layer
+ */
+void NeuralNetwork::readModel() {
+ if (!is_file_exist(model))
+ return;
+ std::ifstream modelFile(model, std::ios::in | std::ios::binary);
+ for (unsigned int i = 0; i < layers.size(); i++)
+ layers[i]->read(modelFile);
+ modelFile.close();
+ std::cout << "read model file \n";
+}
+}
projects / platform / core / ml / nntrainer.git / commitdiff