#include <nntrainer_internal.h>
#include <nntrainer_log.h>
#include <optimizer_factory.h>
-#include <parse_util.h>
#include <sstream>
#include <stdarg.h>
#include <string.h>
include $(CLEAR_VARS)
endif #ENABLE_TFLITE_BACKBONE
-NNTRAINER_SRCS := $(NNTRAINER_ROOT)/nntrainer/src/neuralnet.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/tensor.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/lazy_tensor.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/layer_factory.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/input_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/fc_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/bn_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/loss_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/databuffer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/databuffer_factory.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/databuffer_func.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/databuffer_file.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/util_func.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/optimizer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/parse_util.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/tensor_dim.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/conv2d_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/pooling2d_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/activation_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/flatten_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/model_loader.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/addition_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/concat_layer.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/blas_interface.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/weight.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/adam.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/sgd.cpp \
- $(NNTRAINER_ROOT)/nntrainer/src/optimizer_factory.cpp
+NNTRAINER_SRCS := $(NNTRAINER_ROOT)/nntrainer/models/neuralnet.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/models/model_loader.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/dataset/databuffer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/dataset/databuffer_factory.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/dataset/databuffer_func.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/dataset/databuffer_file.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/tensor/tensor.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/tensor/lazy_tensor.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/tensor/weight.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/tensor/tensor_dim.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/tensor/blas_interface.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/layer_factory.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/input_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/fc_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/bn_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/loss_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/conv2d_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/pooling2d_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/activation_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/flatten_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/addition_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/layers/concat_layer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers/optimizer.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers/adam.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers/sgd.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers/optimizer_factory.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/utils/util_func.cpp \
+ $(NNTRAINER_ROOT)/nntrainer/utils/parse_util.cpp
# Add tflite backbone building
ifeq ($(ENABLE_TFLITE_BACKBONE),1)
-NNTRAINER_SRCS += $(NNTRAINER_ROOT)/nntrainer/src/tflite_layer.cpp
+NNTRAINER_SRCS += $(NNTRAINER_ROOT)/nntrainer/layers/tflite_layer.cpp
endif #ENABLE_TFLITE_BACKBONE
-NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer/include \
+NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer \
+ $(NNTRAINER_ROOT)/nntrainer/dataset \
+ $(NNTRAINER_ROOT)/nntrainer/layers \
+ $(NNTRAINER_ROOT)/nntrainer/models \
+ $(NNTRAINER_ROOT)/nntrainer/tensor \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers \
+ $(NNTRAINER_ROOT)/nntrainer/utils \
$(NNTRAINER_ROOT)/api \
$(NNTRAINER_ROOT)/api/ccapi/include \
$(NNTRAINER_ROOT)/api/capi/include/platform
CAPI_NNTRAINER_SRCS := $(NNTRAINER_ROOT)/api/capi/src/nntrainer.cpp \
$(NNTRAINER_ROOT)/api/capi/src/nntrainer_util.cpp
-CAPI_NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer/include \
+CAPI_NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer \
+ $(NNTRAINER_ROOT)/nntrainer/dataset \
+ $(NNTRAINER_ROOT)/nntrainer/layers \
+ $(NNTRAINER_ROOT)/nntrainer/models \
+ $(NNTRAINER_ROOT)/nntrainer/tensor \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers \
$(NNTRAINER_ROOT)/api \
$(NNTRAINER_ROOT)/api/ccapi/include \
$(NNTRAINER_ROOT)/api/capi/include \
CCAPI_NNTRAINER_SRCS := $(NNTRAINER_ROOT)/api/ccapi/src/factory.cpp
-CCAPI_NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer/include \
+CCAPI_NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer \
+ $(NNTRAINER_ROOT)/nntrainer/dataset \
+ $(NNTRAINER_ROOT)/nntrainer/layers \
+ $(NNTRAINER_ROOT)/nntrainer/models \
+ $(NNTRAINER_ROOT)/nntrainer/tensor \
+ $(NNTRAINER_ROOT)/nntrainer/optimizers \
$(NNTRAINER_ROOT)/api \
$(NNTRAINER_ROOT)/api/ccapi/include \
$(NNTRAINER_ROOT)/api/capi/include/platform
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file databuffer.cpp
+ * @date 04 December 2019
+ * @brief This is buffer object to handle big data
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <cassert>
+#include <climits>
+#include <condition_variable>
+#include <cstring>
+#include <databuffer.h>
+#include <databuffer_util.h>
+#include <functional>
+#include <iomanip>
+#include <mutex>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <sstream>
+#include <stdexcept>
+#include <stdio.h>
+#include <stdlib.h>
+#include <thread>
+#include <util_func.h>
+
+std::exception_ptr globalExceptionPtr = nullptr;
+
+namespace nntrainer {
+
+constexpr char USER_DATA[] = "user_data";
+
+std::mutex data_lock;
+
+std::mutex readyTrainData;
+std::mutex readyValData;
+std::mutex readyTestData;
+
+std::condition_variable cv_train;
+std::condition_variable cv_val;
+std::condition_variable cv_test;
+
+DataBuffer::DataBuffer(DataBufferType type) :
+ train_running(),
+ val_running(),
+ test_running(),
+ train_thread(),
+ val_thread(),
+ test_thread(),
+ data_buffer_type(type),
+ user_data(nullptr) {
+ SET_VALIDATION(false);
+ class_num = 0;
+ cur_train_bufsize = 0;
+ cur_val_bufsize = 0;
+ cur_test_bufsize = 0;
+ train_bufsize = 0;
+ val_bufsize = 0;
+ test_bufsize = 0;
+ max_train = 0;
+ max_val = 0;
+ max_test = 0;
+ rest_train = 0;
+ rest_val = 0;
+ rest_test = 0;
+ batch_size = 0;
+ train_running = false;
+ val_running = false;
+ test_running = false;
+ trainReadyFlag = DATA_NOT_READY;
+ valReadyFlag = DATA_NOT_READY;
+ testReadyFlag = DATA_NOT_READY;
+ rng.seed(getSeed());
+};
+
+int DataBuffer::rangeRandom(int min, int max) {
+ std::uniform_int_distribution<int> dist(min, max);
+ return dist(rng);
+}
+
+int DataBuffer::run(BufferType type) {
+ int status = ML_ERROR_NONE;
+ switch (type) {
+ case BufferType::BUF_TRAIN:
+ if (trainReadyFlag == DATA_ERROR)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ if (validation[DATA_TRAIN]) {
+ this->train_running = true;
+ this->train_thread = std::thread(&DataBuffer::updateData, this, type);
+ if (globalExceptionPtr) {
+ try {
+ std::rethrow_exception(globalExceptionPtr);
+ } catch (const std::exception &ex) {
+ std::cout << ex.what() << "\n";
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ }
+ } else {
+ ml_loge("Error: Training Data Set is not valid");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ break;
+ case BufferType::BUF_VAL:
+ if (valReadyFlag == DATA_ERROR)
+ return ML_ERROR_INVALID_PARAMETER;
+ if (validation[DATA_VAL]) {
+ this->val_running = true;
+ this->val_thread = std::thread(&DataBuffer::updateData, this, type);
+ if (globalExceptionPtr) {
+ try {
+ std::rethrow_exception(globalExceptionPtr);
+ } catch (const std::exception &ex) {
+ std::cout << ex.what() << "\n";
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ }
+ } else {
+ ml_loge("Error: Validation Data Set is not valid");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ break;
+ case BufferType::BUF_TEST:
+ if (testReadyFlag == DATA_ERROR)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ if (validation[DATA_TEST]) {
+ this->test_running = true;
+ this->test_thread = std::thread(&DataBuffer::updateData, this, type);
+ if (globalExceptionPtr) {
+ try {
+ std::rethrow_exception(globalExceptionPtr);
+ } catch (const std::exception &ex) {
+ std::cout << ex.what() << "\n";
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ }
+ } else {
+ ml_loge("Error: Test Data Set is not valid");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ break;
+ default:
+ ml_loge("Error: Not Supported Data Type");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+
+ return status;
+}
+
+int DataBuffer::clear(BufferType type) {
+ int status = ML_ERROR_NONE;
+ NN_EXCEPTION_NOTI(DATA_NOT_READY);
+ switch (type) {
+ case BufferType::BUF_TRAIN: {
+ train_running = false;
+ if (validation[DATA_TRAIN] && true == train_thread.joinable())
+ train_thread.join();
+ this->train_data.clear();
+ this->train_data_label.clear();
+ this->cur_train_bufsize = 0;
+ this->rest_train = max_train;
+ } break;
+ case BufferType::BUF_VAL: {
+ val_running = false;
+ if (validation[DATA_VAL] && true == val_thread.joinable())
+ val_thread.join();
+ this->val_data.clear();
+ this->val_data_label.clear();
+ this->cur_val_bufsize = 0;
+ this->rest_val = max_val;
+ } break;
+ case BufferType::BUF_TEST: {
+ test_running = false;
+ if (validation[DATA_TEST] && true == test_thread.joinable())
+ test_thread.join();
+ this->test_data.clear();
+ this->test_data_label.clear();
+ this->cur_test_bufsize = 0;
+ this->rest_test = max_test;
+ } break;
+ default:
+ ml_loge("Error: Not Supported Data Type");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+ return status;
+}
+
+int DataBuffer::clear() {
+ unsigned int i;
+
+ int status = ML_ERROR_NONE;
+ for (i = (int)BufferType::BUF_TRAIN; i <= (int)BufferType::BUF_TEST; ++i) {
+ BufferType type = static_cast<BufferType>(i);
+ status = this->clear(type);
+
+ if (status != ML_ERROR_NONE) {
+ ml_loge("Error: error occurred during clearing");
+ return status;
+ }
+ }
+
+ return status;
+}
+
+bool DataBuffer::getDataFromBuffer(BufferType type, float *out, float *label) {
+
+ using QueueType = std::vector<std::vector<float>>;
+
+ auto wait_for_data_fill = [](std::mutex &ready_mutex,
+ std::condition_variable &cv, DataStatus &flag,
+ const unsigned int batch_size,
+ QueueType &queue) {
+ while (true) {
+ std::unique_lock<std::mutex> ul(ready_mutex);
+ cv.wait(ul, [&]() -> bool { return flag; });
+ if (flag == DATA_ERROR || flag == DATA_END)
+ return queue.size() < batch_size ? false : true;
+
+ if (flag == DATA_READY && queue.size() >= batch_size)
+ return true;
+ }
+
+ throw std::logic_error("[getDataFromBuffer] control should not reach here");
+ };
+
+ auto fill_bundled_data_from_queue =
+ [](std::mutex &q_lock, QueueType &q, const unsigned int batch_size,
+ const unsigned int feature_size, float *buf) {
+ for (unsigned int b = 0; b < batch_size; ++b)
+ std::copy(q[b].begin(), q[b].begin() + feature_size,
+ buf + b * feature_size);
+
+ q_lock.lock();
+ q.erase(q.begin(), q.begin() + batch_size);
+ q_lock.unlock();
+ };
+
+ /// facade that wait for the databuffer to be filled and pass it to outparam
+ /// note that batch_size is passed as an argument because it can vary by
+ /// BufferType::BUF_TYPE later...
+ auto fill_out_params =
+ [&](std::mutex &ready_mutex, std::condition_variable &cv, DataStatus &flag,
+ QueueType &data_q, QueueType &label_q, const unsigned int batch_size,
+ unsigned int &cur_bufsize) {
+ if (!wait_for_data_fill(ready_mutex, cv, flag, batch_size, data_q)) {
+ return false;
+ }
+
+ fill_bundled_data_from_queue(data_lock, data_q, batch_size,
+ this->input_dim.getFeatureLen(), out);
+ fill_bundled_data_from_queue(data_lock, label_q, batch_size,
+ this->class_num, label);
+
+ cur_bufsize -= batch_size;
+ return true;
+ };
+
+ switch (type) {
+ case BufferType::BUF_TRAIN:
+ if (!fill_out_params(readyTrainData, cv_train, trainReadyFlag, train_data,
+ train_data_label, batch_size, cur_train_bufsize))
+ return false;
+ break;
+ case BufferType::BUF_VAL:
+ if (!fill_out_params(readyValData, cv_val, valReadyFlag, val_data,
+ val_data_label, batch_size, cur_val_bufsize))
+ return false;
+ break;
+ case BufferType::BUF_TEST:
+ if (!fill_out_params(readyTestData, cv_test, testReadyFlag, test_data,
+ test_data_label, batch_size, cur_test_bufsize))
+ return false;
+ break;
+ default:
+ ml_loge("Error: Not Supported Data Type");
+ return false;
+ break;
+ }
+
+ return true;
+}
+
+int DataBuffer::setClassNum(unsigned int num) {
+ int status = ML_ERROR_NONE;
+ if (num <= 0) {
+ ml_loge("Error: number of class should be bigger than 0");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ if (class_num != 0 && class_num != num) {
+ ml_loge("Error: number of class should be same with number of label label");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ class_num = num;
+ return status;
+}
+
+int DataBuffer::setBufSize(unsigned int size) {
+ int status = ML_ERROR_NONE;
+ train_bufsize = size;
+ val_bufsize = size;
+ test_bufsize = size;
+ return status;
+}
+
+int DataBuffer::setBatchSize(unsigned int size) {
+ int status = ML_ERROR_NONE;
+ if (size == 0) {
+ ml_loge("Error: batch size must be greater than 0");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ batch_size = size;
+ return status;
+}
+
+int DataBuffer::init() {
+ if (batch_size == 0) {
+ ml_loge("Error: batch size must be greater than 0");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ /** for now, train_bufsize, val_bufsize and test_bufsize are same value */
+ if (train_bufsize < batch_size) {
+ if (train_bufsize > 1) {
+ ml_logw("Dataset buffer size reset to be at least batch size");
+ }
+ train_bufsize = batch_size;
+ val_bufsize = batch_size;
+ test_bufsize = batch_size;
+ }
+
+ if (!class_num) {
+ ml_loge("Error: number of class must be set");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ if (!this->input_dim.getFeatureLen()) {
+ ml_loge("Error: feature size must be set");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ this->cur_train_bufsize = 0;
+ this->cur_val_bufsize = 0;
+ this->cur_test_bufsize = 0;
+
+ readyTrainData.lock();
+ trainReadyFlag = DATA_NOT_READY;
+ readyTrainData.unlock();
+ readyValData.lock();
+ valReadyFlag = DATA_NOT_READY;
+ readyValData.unlock();
+ readyTestData.lock();
+ testReadyFlag = DATA_NOT_READY;
+ readyTestData.unlock();
+ return ML_ERROR_NONE;
+}
+
+int DataBuffer::setFeatureSize(TensorDim indim) {
+ int status = ML_ERROR_NONE;
+ input_dim = indim;
+ return status;
+}
+
+void DataBuffer::displayProgress(const int count, BufferType type, float loss) {
+ int barWidth = 20;
+ float max_size = max_train;
+ switch (type) {
+ case BufferType::BUF_TRAIN:
+ max_size = max_train;
+ break;
+ case BufferType::BUF_VAL:
+ max_size = max_val;
+ break;
+ case BufferType::BUF_TEST:
+ max_size = max_test;
+ break;
+ default:
+ ml_loge("Error: Not Supported Data Type");
+ break;
+ }
+ std::stringstream ssInt;
+ ssInt << count * batch_size;
+
+ std::string str = ssInt.str();
+ int len = str.length();
+
+ if (max_size == 0) {
+ int pad_left = (barWidth - len) / 2;
+ int pad_right = barWidth - pad_left - len;
+ std::string out_str =
+ std::string(pad_left, ' ') + str + std::string(pad_right, ' ');
+ std::cout << " [ ";
+ std::cout << out_str;
+ std::cout << " ] "
+ << " ( Training Loss: " << loss << " )\r";
+ } else {
+ float progress;
+ if (batch_size > max_size)
+ progress = 1.0;
+ else
+ progress = (((float)(count * batch_size)) / max_size);
+
+ int pos = barWidth * progress;
+ std::cout << " [ ";
+ for (int l = 0; l < barWidth; ++l) {
+ if (l <= pos)
+ std::cout << "=";
+ else
+ std::cout << " ";
+ }
+ std::cout << " ] " << int(progress * 100.0) << "% ( Training Loss: " << loss
+ << " )\r";
+ }
+
+ std::cout.flush();
+}
+
+int DataBuffer::setProperty(std::vector<void *> values) {
+ int status = ML_ERROR_NONE;
+ std::vector<std::string> properties;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ char *key_ptr = (char *)values[i];
+ std::string key = key_ptr;
+ std::string value;
+
+ /** Handle the user_data as a special case */
+ if (key == USER_DATA) {
+ /** This ensures that a valid user_data element is passed by the user */
+ if (i + 1 >= values.size())
+ return ML_ERROR_INVALID_PARAMETER;
+
+ this->user_data = values[i + 1];
+
+ /** As values of i+1 is consumed, increase i by 1 */
+ i++;
+ } else {
+ properties.push_back(key);
+ continue;
+ }
+ }
+
+ status = setProperty(properties);
+
+ return status;
+}
+
+int DataBuffer::setProperty(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+ status = getKeyValue(values[i], key, value);
+ NN_RETURN_STATUS();
+
+ unsigned int type = parseDataProperty(key);
+ if (value.empty())
+ return ML_ERROR_INVALID_PARAMETER;
+
+ status = setProperty(static_cast<PropertyType>(type), value);
+ NN_RETURN_STATUS();
+ }
+
+ return status;
+}
+
+int DataBuffer::setProperty(const PropertyType type, std::string &value) {
+ int status = ML_ERROR_NONE;
+ unsigned int size = 0;
+
+ switch (type) {
+ case PropertyType::buffer_size:
+ status = setUint(size, value);
+ NN_RETURN_STATUS();
+ status = this->setBufSize(size);
+ NN_RETURN_STATUS();
+ break;
+ default:
+ ml_loge("Error: Unknown Data Buffer Property Key");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+
+ return status;
+}
+
+int DataBuffer::setGeneratorFunc(BufferType type, datagen_cb func) {
+ return ML_ERROR_NOT_SUPPORTED;
+}
+
+int DataBuffer::setDataFile(DataType type, std::string path) {
+ return ML_ERROR_NOT_SUPPORTED;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file databuffer.h
+ * @date 04 December 2019
+ * @brief This is buffer object to handle big data
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __DATABUFFER_H__
+#define __DATABUFFER_H__
+#ifdef __cplusplus
+
+#include <memory>
+#include <random>
+#include <thread>
+#include <vector>
+
+#include <dataset.h>
+#include <tensor_dim.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Aliasing from ccapi ml::train
+ */
+using DataBufferType = ml::train::DatasetType;
+using DatasetDataType = ml::train::DatasetDataType;
+using datagen_cb = ml::train::datagen_cb;
+
+/**
+ * @brief Enumeration of data type
+ */
+typedef enum {
+ DATA_TRAIN =
+ (int)ml::train::DatasetDataType::DATA_TRAIN, /** data for training */
+ DATA_VAL =
+ (int)ml::train::DatasetDataType::DATA_VAL, /** data for validation */
+ DATA_TEST = (int)ml::train::DatasetDataType::DATA_TEST, /** data for test */
+ DATA_LABEL = (int)ml::train::DatasetDataType::DATA_LABEL, /** label names */
+ DATA_UNKNOWN =
+ (int)ml::train::DatasetDataType::DATA_UNKNOWN /** data not known */
+} DataType;
+
+/**
+ * @brief Enumeration of buffer type
+ */
+enum class BufferType {
+ BUF_TRAIN = DATA_TRAIN, /** BUF_TRAIN ( Buffer for training ) */
+ BUF_VAL = DATA_VAL, /** BUF_VAL ( Buffer for validation ) */
+ BUF_TEST = DATA_TEST, /** BUF_TEST ( Buffer for test ) */
+ BUF_UNKNOWN = DATA_UNKNOWN /** BUF_UNKNOWN ( unknown ) */
+};
+
+/**
+ * @class DataBuffer Data Buffers
+ * @brief Data Buffer for read and manage data
+ */
+class DataBuffer : public ml::train::Dataset {
+public:
+ /**
+ * @brief Create Buffer
+ * @retval DataBuffer
+ */
+ DataBuffer(DataBufferType type);
+
+ /**
+ * @brief Initialize Buffer with data buffer private variables
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int init();
+
+ /**
+ * @brief Update Data Buffer ( it is for child thread )
+ * @param[in] BufferType training, validation, test
+ * @retval void
+ */
+ virtual void updateData(BufferType type) = 0;
+
+ /**
+ * @brief function for thread ( training, validation, test )
+ * @param[in] BufferType training, validation, test
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int run(BufferType type);
+
+ /**
+ * @brief clear thread ( training, validation, test )
+ * @param[in] BufferType training, validation, test
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int clear(BufferType type);
+
+ /**
+ * @brief clear all thread ( training, validation, test )
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int clear();
+
+ /**
+ * @brief get Data from Data Buffer using databuffer param
+ * @param[in] BufferType training, validation, test
+ * @param[out] out feature data ( batch_size size ), a contiguous and
+ * allocated memory block should be passed
+ * @param[out] label label data ( batch_size size ), a contiguous and
+ * allocated memory block should be passed
+ * @retval true/false
+ */
+ bool getDataFromBuffer(BufferType type, float *out, float *label);
+
+ /**
+ * @brief set number of class
+ * @param[in] number of class
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setClassNum(unsigned int n);
+
+ /**
+ * @brief set buffer size
+ * @param[in] buffer size
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setBufSize(unsigned int n);
+
+ /**
+ * @brief set batch size
+ * @param[in] n batch size
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setBatchSize(unsigned int n);
+
+ /**
+ * @brief set feature size
+ * @param[in] feature batch size. It is equal to input layer's hidden size
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setFeatureSize(TensorDim indim);
+
+ /**
+ * @brief set feature size
+ * @retval max_train
+ */
+ unsigned int getMaxTrain() { return max_train; }
+
+ /**
+ * @brief set feature size
+ * @retval max_val
+ */
+ unsigned int getMaxVal() { return max_val; }
+
+ /**
+ * @brief set feature size
+ * @retval max_test
+ */
+ unsigned int getMaxTest() { return max_test; }
+
+ /**
+ * @brief Display Progress
+ * @param[in] count calculated set ( batch_size size )
+ * @param[in] type buffer type ( BUF_TRAIN, BUF_VAL, BUF_TEST )
+ * @retval void
+ */
+ void displayProgress(const int count, BufferType type, float loss);
+
+ /**
+ * @brief return validation of data set
+ * @retval validation
+ */
+ bool *getValidation() { return validation; }
+
+ /**
+ * @brief set property
+ * @param[in] values values of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(std::vector<std::string> values);
+
+ /**
+ * @brief set property to allow setting user_data for cb
+ * @param[in] values values of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(std::vector<void *> values);
+
+ /**
+ * @brief set function pointer for each type
+ * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
+ * @param[in] call back function pointer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setGeneratorFunc(DatasetDataType type, datagen_cb func) {
+ return setGeneratorFunc((BufferType)type, func);
+ }
+
+ /**
+ * @brief set function pointer for each type
+ * @param[in] type Buffer Type
+ * @param[in] call back function pointer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setGeneratorFunc(BufferType type, datagen_cb func);
+
+ /**
+ * @brief set train data file name
+ * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
+ * @param[in] path file path
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setDataFile(DatasetDataType type, std::string path) {
+ return setDataFile((DataType)type, path);
+ }
+
+ /**
+ * @brief set train data file name
+ * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
+ * @param[in] path file path
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setDataFile(DataType type, std::string path);
+
+ enum class PropertyType {
+ train_data = 0,
+ val_data = 1,
+ test_data = 2,
+ label_data = 3,
+ buffer_size = 4,
+ unknown = 5
+ };
+
+protected:
+ /**
+ * @brief Number of Data Set
+ */
+ static constexpr const unsigned int NBUFTYPE = 4;
+
+ /**
+ * @brief state of the data buffer while getting the data
+ */
+ typedef enum {
+ DATA_NOT_READY = 0,
+ DATA_READY = 1,
+ DATA_END = 2,
+ DATA_ERROR = 3,
+ } DataStatus;
+
+ /**
+ * @brief status of thread
+ */
+ DataStatus trainReadyFlag;
+ DataStatus valReadyFlag;
+ DataStatus testReadyFlag;
+
+ /**
+ * @brief Data Queues for each data set
+ */
+ std::vector<std::vector<float>> train_data;
+ std::vector<std::vector<float>> train_data_label;
+ std::vector<std::vector<float>> val_data;
+ std::vector<std::vector<float>> val_data_label;
+ std::vector<std::vector<float>> test_data;
+ std::vector<std::vector<float>> test_data_label;
+
+ /**
+ * @brief feature size
+ */
+ TensorDim input_dim;
+
+ /**
+ * @brief number of class
+ */
+ unsigned int class_num;
+
+ /**
+ * @brief number of remain data for each data queue
+ */
+ unsigned int cur_train_bufsize;
+ unsigned int cur_val_bufsize;
+ unsigned int cur_test_bufsize;
+
+ /**
+ * @brief queue size for each data set
+ */
+ unsigned int train_bufsize;
+ unsigned int val_bufsize;
+ unsigned int test_bufsize;
+
+ unsigned int max_train;
+ unsigned int max_val;
+ unsigned int max_test;
+
+ /**
+ * @brief remain data set size
+ */
+ unsigned int rest_train;
+ unsigned int rest_val;
+ unsigned int rest_test;
+
+ /**
+ * @brief batch size
+ */
+ unsigned int batch_size;
+
+ /**
+ * @brief flags to check status
+ */
+ bool train_running;
+ bool val_running;
+ bool test_running;
+
+ /**
+ * @brief ids to check duplication
+ */
+ std::vector<unsigned int> train_mark;
+ std::vector<unsigned int> val_mark;
+ std::vector<unsigned int> test_mark;
+
+ /**
+ * @brief threads to generate data for queue
+ */
+ std::thread train_thread;
+ std::thread val_thread;
+ std::thread test_thread;
+
+ std::vector<std::string> labels;
+ bool validation[NBUFTYPE];
+
+ /**
+ * @brief return random int value between min to max
+ * @param[in] min minimum vaule
+ * @param[in] max maximum value
+ * @retval int return value
+ */
+ int rangeRandom(int min, int max);
+
+ std::mt19937 rng;
+
+ /**
+ * @brief The type of data buffer
+ */
+ DataBufferType data_buffer_type;
+
+ /** The user_data to be used for the data generator callback */
+ void *user_data;
+
+ /**
+ * @brief set property
+ * @param[in] type type of property
+ * @param[in] value string value of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setProperty(const DataBuffer::PropertyType type,
+ std::string &value);
+};
+
+} // namespace nntrainer
+#endif /* __cplusplus */
+#endif /* __DATABUFFER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file optimizer_factory.cpp
+ * @date 11 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the databuffer factory.
+ */
+
+#include <databuffer_factory.h>
+
+#include <databuffer_file.h>
+#include <databuffer_func.h>
+#include <nntrainer_error.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Factory creator with constructor
+ */
+std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type) {
+ switch (type) {
+ case DataBufferType::GENERATOR:
+ return std::make_unique<DataBufferFromCallback>();
+ case DataBufferType::FILE:
+ return std::make_unique<DataBufferFromDataFile>();
+ case DataBufferType::UNKNOWN:
+ /** fallthrough intended */
+ default:
+ throw std::invalid_argument("Unknown type for the dataset");
+ }
+}
+
+/**
+ * @brief Factory creator with constructor for dataset
+ */
+std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
+ const char *train_file,
+ const char *valid_file,
+ const char *test_file) {
+ if (type != DataBufferType::FILE)
+ throw std::invalid_argument(
+ "Cannot create dataset with files with the given dataset type");
+
+ std::unique_ptr<DataBuffer> dataset = createDataBuffer(type);
+
+ if (dataset->setDataFile(DataType::DATA_TRAIN, train_file) != ML_ERROR_NONE)
+ throw std::invalid_argument("Invalid train file");
+
+ if (valid_file &&
+ dataset->setDataFile(DataType::DATA_VAL, valid_file) != ML_ERROR_NONE)
+ throw std::invalid_argument("Invalid valid file");
+
+ if (test_file &&
+ dataset->setDataFile(DataType::DATA_TEST, test_file) != ML_ERROR_NONE)
+ throw std::invalid_argument("Invalid test file");
+
+ return dataset;
+}
+
+/**
+ * @brief Factory creator with constructor for dataset
+ */
+std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
+ datagen_cb train, datagen_cb valid,
+ datagen_cb test) {
+ if (type != DataBufferType::GENERATOR)
+ throw std::invalid_argument("Cannot create dataset with generator "
+ "callbacks with the given dataset type");
+
+ std::unique_ptr<DataBuffer> dataset = createDataBuffer(type);
+
+ if (dataset->setGeneratorFunc((BufferType)DataType::DATA_TRAIN, train) !=
+ ML_ERROR_NONE)
+ throw std::invalid_argument("Invalid train data generator");
+
+ if (valid && dataset->setGeneratorFunc((BufferType)DataType::DATA_VAL,
+ valid) != ML_ERROR_NONE)
+ throw std::invalid_argument("Invalid valid data generator");
+
+ if (test && dataset->setGeneratorFunc((BufferType)DataType::DATA_TEST,
+ test) != ML_ERROR_NONE)
+ throw std::invalid_argument("Invalid test data generator");
+
+ return dataset;
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file databuffer_factory.h
+ * @date 19 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the layer factory.
+ */
+
+#ifndef __DATABUFFER_FACTORY_H__
+#define __DATABUFFER_FACTORY_H__
+#ifdef __cplusplus
+
+#include <databuffer.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Factory creator with constructor
+ */
+std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type);
+
+/**
+ * @brief Factory creator with constructor for databuffer with files
+ */
+std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
+ const char *train_file,
+ const char *valid_file = nullptr,
+ const char *test_file = nullptr);
+
+/**
+ * @brief Factory creator with constructor for databuffer with callbacks
+ */
+std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
+ datagen_cb train,
+ datagen_cb valid = nullptr,
+ datagen_cb test = nullptr);
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __DATABUFFER_FACTORY_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file databuffer_file.cpp
+ * @date 27 April 2020
+ * @brief This is buffer object take data from raw files
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <assert.h>
+#include <climits>
+#include <condition_variable>
+#include <cstring>
+#include <databuffer_file.h>
+#include <databuffer_util.h>
+#include <functional>
+#include <iomanip>
+#include <mutex>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <sstream>
+#include <stdexcept>
+#include <stdio.h>
+#include <stdlib.h>
+#include <thread>
+
+extern std::exception_ptr globalExceptionPtr;
+
+namespace nntrainer {
+
+extern std::mutex data_lock;
+
+extern std::mutex readyTrainData;
+extern std::mutex readyValData;
+extern std::mutex readyTestData;
+
+extern std::condition_variable cv_train;
+extern std::condition_variable cv_val;
+extern std::condition_variable cv_test;
+
+static long getFileSize(std::string file_name) {
+ std::ifstream file_stream(file_name.c_str(), std::ios::in | std::ios::binary);
+ if (file_stream.good()) {
+ file_stream.seekg(0, std::ios::end);
+ return file_stream.tellg();
+ } else {
+ return 0;
+ }
+}
+
+int DataBufferFromDataFile::init() {
+ int status = ML_ERROR_NONE;
+
+ status = DataBuffer::init();
+ if (status != ML_ERROR_NONE)
+ return status;
+
+ if (validation[DATA_TRAIN] && max_train < batch_size) {
+ max_train = batch_size;
+ }
+
+ if (validation[DATA_VAL] && max_val < batch_size) {
+ max_val = batch_size;
+ }
+
+ if (validation[DATA_TEST] && max_test < batch_size) {
+ max_test = batch_size;
+ }
+
+ this->rest_train = max_train;
+ this->rest_val = max_val;
+ this->rest_test = max_test;
+
+ this->train_running = true;
+ this->val_running = true;
+ this->test_running = true;
+
+ if (validation[DATA_TRAIN] && max_train < train_bufsize) {
+ ml_logw("Warning: Total number of train is less than train buffer size. "
+ "Train buffer size is set as total number of train");
+ train_bufsize = batch_size;
+ }
+
+ if (validation[DATA_VAL] && max_val < val_bufsize) {
+ ml_logw(
+ "Warning: Total number of validation is less than validation buffer "
+ "size. Validation buffer size is set as total number of validation");
+ val_bufsize = batch_size;
+ }
+
+ if (validation[DATA_TEST] && max_test < test_bufsize) {
+ ml_logw("Warning: Total number of test is less than test buffer size. Test "
+ "buffer size is set as total number of test");
+ test_bufsize = batch_size;
+ }
+
+ return ML_ERROR_NONE;
+}
+
+void DataBufferFromDataFile::updateData(BufferType type) {
+ unsigned int max_size = 0;
+ unsigned int buf_size = 0;
+ unsigned int *rest_size = NULL;
+ unsigned int *cur_size = NULL;
+ bool *running = NULL;
+ std::vector<std::vector<float>> *data = NULL;
+ std::vector<std::vector<float>> *datalabel = NULL;
+ std::ifstream file;
+ switch (type) {
+ case BufferType::BUF_TRAIN: {
+ max_size = max_train;
+ buf_size = train_bufsize;
+ rest_size = &rest_train;
+ cur_size = &cur_train_bufsize;
+ running = &train_running;
+ data = &train_data;
+ datalabel = &train_data_label;
+
+ std::ifstream train_stream(train_name, std::ios::in | std::ios::binary);
+ file.swap(train_stream);
+ readyTrainData.lock();
+ trainReadyFlag = DATA_NOT_READY;
+ readyTrainData.unlock();
+
+ } break;
+ case BufferType::BUF_VAL: {
+ max_size = max_val;
+ buf_size = val_bufsize;
+ rest_size = &rest_val;
+ cur_size = &cur_val_bufsize;
+ running = &val_running;
+ data = &val_data;
+ datalabel = &val_data_label;
+
+ std::ifstream val_stream(val_name, std::ios::in | std::ios::binary);
+ file.swap(val_stream);
+ readyValData.lock();
+ valReadyFlag = DATA_NOT_READY;
+ readyValData.unlock();
+
+ } break;
+ case BufferType::BUF_TEST: {
+ max_size = max_test;
+ buf_size = test_bufsize;
+ rest_size = &rest_test;
+ cur_size = &cur_test_bufsize;
+ running = &test_running;
+ data = &test_data;
+ datalabel = &test_data_label;
+
+ std::ifstream test_stream(test_name, std::ios::in | std::ios::binary);
+ file.swap(test_stream);
+ readyTestData.lock();
+ testReadyFlag = DATA_NOT_READY;
+ readyTestData.unlock();
+ } break;
+ default:
+ try {
+ throw std::runtime_error("Error: Not Supported Data Type");
+ } catch (...) {
+ globalExceptionPtr = std::current_exception();
+ NN_EXCEPTION_NOTI(DATA_ERROR);
+ return;
+ }
+ break;
+ }
+
+ unsigned int I;
+ std::vector<unsigned int> mark;
+ mark.resize(max_size);
+ file.clear();
+ file.seekg(0, std::ios_base::end);
+ uint64_t file_length = file.tellg();
+
+ for (unsigned int i = 0; i < max_size; ++i) {
+ mark[i] = i;
+ }
+
+ while ((*running)) {
+
+ if (mark.size() == 0) {
+ NN_EXCEPTION_NOTI(DATA_END);
+ break;
+ }
+
+ if (buf_size - (*cur_size) > 0 && (*rest_size) > 0) {
+ std::vector<float> vec;
+ std::vector<float> veclabel;
+
+ unsigned int id = rangeRandom(0, mark.size() - 1);
+ I = mark[id];
+
+ try {
+ if (I > max_size) {
+ throw std::runtime_error(
+ "Error: Test case id cannot exceed maximum number of test");
+ }
+ } catch (...) {
+ globalExceptionPtr = std::current_exception();
+ NN_EXCEPTION_NOTI(DATA_ERROR);
+ return;
+ }
+
+ mark.erase(mark.begin() + id);
+
+ uint64_t position =
+ (uint64_t)((I * input_dim.getFeatureLen() + (uint64_t)I * class_num) *
+ sizeof(float));
+ try {
+ if (position > file_length) {
+ throw std::runtime_error("Error: Cannot exceed max file size");
+ }
+ } catch (...) {
+ globalExceptionPtr = std::current_exception();
+ NN_EXCEPTION_NOTI(DATA_ERROR);
+ return;
+ }
+
+ file.seekg(position, std::ios::beg);
+
+ for (unsigned int j = 0; j < input_dim.getFeatureLen(); ++j) {
+ float d;
+ file.read((char *)&d, sizeof(float));
+ vec.push_back(d);
+ }
+
+ for (unsigned int j = 0; j < class_num; ++j) {
+ float d;
+ file.read((char *)&d, sizeof(float));
+ veclabel.push_back(d);
+ }
+
+ data_lock.lock();
+ data->push_back(vec);
+ datalabel->push_back(veclabel);
+ (*rest_size)--;
+ (*cur_size)++;
+ data_lock.unlock();
+ }
+ if (buf_size == (*cur_size)) {
+ NN_EXCEPTION_NOTI(DATA_READY);
+ }
+ }
+
+ file.close();
+}
+
+int DataBufferFromDataFile::setDataFile(DataType type, std::string path) {
+ int status = ML_ERROR_NONE;
+ std::ifstream data_file(path.c_str());
+
+ switch (type) {
+ case DATA_TRAIN: {
+ validation[type] = true;
+ if (!data_file.good()) {
+ ml_loge(
+ "Error: Cannot open data file, Datafile is necessary for training");
+ validation[type] = false;
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ train_name = path;
+ } break;
+ case DATA_VAL: {
+ validation[type] = true;
+ if (!data_file.good()) {
+ ml_loge("Error: Cannot open validation data file. Cannot validate "
+ "training result");
+ validation[type] = false;
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ val_name = path;
+ } break;
+ case DATA_TEST: {
+ validation[type] = true;
+ if (!data_file.good()) {
+ ml_loge("Error: Cannot open test data file. Cannot test training result");
+ validation[type] = false;
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ test_name = path;
+ } break;
+ case DATA_LABEL: {
+ std::string data;
+ validation[type] = true;
+ if (!data_file.good()) {
+ ml_loge("Error: Cannot open label file");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ while (data_file >> data) {
+ labels.push_back(data);
+ }
+ if (class_num != 0 && class_num != labels.size()) {
+ ml_loge("Error: number of label should be same with number class number");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ class_num = labels.size();
+ } break;
+ case DATA_UNKNOWN:
+ default:
+ ml_loge("Error: Not Supported Data Type");
+ SET_VALIDATION(false);
+ return ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+ return status;
+}
+
+int DataBufferFromDataFile::setFeatureSize(TensorDim tdim) {
+ int status = ML_ERROR_NONE;
+ long file_size = 0;
+
+ status = DataBuffer::setFeatureSize(tdim);
+ if (status != ML_ERROR_NONE)
+ return status;
+
+ if (validation[DATA_TRAIN]) {
+ file_size = getFileSize(train_name);
+ max_train = static_cast<unsigned int>(
+ file_size /
+ (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
+ if (max_train < batch_size) {
+ ml_logw("Warning: number of training data is smaller than batch size");
+ }
+ } else {
+ max_train = 0;
+ }
+
+ if (validation[DATA_VAL]) {
+ file_size = getFileSize(val_name);
+ max_val = static_cast<unsigned int>(
+ file_size /
+ (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
+ if (max_val < batch_size) {
+ ml_logw("Warning: number of val data is smaller than batch size");
+ }
+ } else {
+ max_val = 0;
+ }
+
+ if (validation[DATA_TEST]) {
+ file_size = getFileSize(test_name);
+ max_test = static_cast<unsigned int>(
+ file_size /
+ (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
+ if (max_test < batch_size) {
+ ml_logw("Warning: number of test data is smaller than batch size");
+ }
+ } else {
+ max_test = 0;
+ }
+ return status;
+}
+
+int DataBufferFromDataFile::setProperty(const PropertyType type,
+ std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ if (data_buffer_type != DataBufferType::FILE)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ switch (type) {
+ case PropertyType::train_data:
+ status = this->setDataFile(DATA_TRAIN, value);
+ break;
+ case PropertyType::val_data:
+ status = this->setDataFile(DATA_VAL, value);
+ break;
+ case PropertyType::test_data:
+ status = this->setDataFile(DATA_TEST, value);
+ break;
+ case PropertyType::label_data:
+ status = this->setDataFile(DATA_LABEL, value);
+ break;
+ default:
+ status = DataBuffer::setProperty(type, value);
+ break;
+ }
+
+ return status;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file databuffer_file.h
+ * @date 27 April 2020
+ * @brief This is buffer object take data from raw files
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __DATABUFFER_FILE_H__
+#define __DATABUFFER_FILE_H__
+#ifdef __cplusplus
+
+#include <fstream>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include <databuffer.h>
+
+namespace nntrainer {
+
+/**
+ * @class DataBufferFromDataFile Data Buffer from Raw Data File
+ * @brief Data Buffer from reading raw data
+ */
+class DataBufferFromDataFile : public DataBuffer {
+
+public:
+ /**
+ * @brief Constructor
+ */
+ DataBufferFromDataFile() : DataBuffer(DataBufferType::FILE) {}
+
+ /**
+ * @brief Destructor
+ */
+ ~DataBufferFromDataFile() = default;
+
+ /**
+ * @brief Initialize Buffer with data buffer private variables
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int init();
+
+ /**
+ * @brief Update Data Buffer ( it is for child thread )
+ * @param[in] BufferType training, validation, test
+ * @retval void
+ */
+ void updateData(BufferType type);
+
+ /**
+ * @brief set train data file name
+ * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
+ * @param[in] path file path
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setDataFile(DataType type, std::string path);
+
+ /**
+ * @brief set feature size
+ * @param[in] Input Tensor Dimension.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setFeatureSize(TensorDim indim);
+
+ /**
+ * @brief set property
+ * @param[in] type type of property
+ * @param[in] value string value of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(const PropertyType type, std::string &value);
+
+private:
+ /**
+ * @brief raw data file names
+ */
+ std::string train_name;
+ std::string val_name;
+ std::string test_name;
+};
+
+} // namespace nntrainer
+#endif /* __cplusplus */
+#endif /* __DATABUFFER_FILE_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file databuffer_file.cpp
+ * @date 27 April 2020
+ * @brief This is buffer object take data from raw files
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <assert.h>
+#include <climits>
+#include <condition_variable>
+#include <cstring>
+#include <databuffer_func.h>
+#include <databuffer_util.h>
+#include <functional>
+#include <iomanip>
+#include <mutex>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <sstream>
+#include <stdexcept>
+#include <stdio.h>
+#include <stdlib.h>
+#include <thread>
+
+extern std::exception_ptr globalExceptionPtr;
+
+namespace nntrainer {
+
+extern std::mutex data_lock;
+
+extern std::mutex readyTrainData;
+extern std::mutex readyValData;
+extern std::mutex readyTestData;
+
+extern std::condition_variable cv_train;
+extern std::condition_variable cv_val;
+extern std::condition_variable cv_test;
+
+int DataBufferFromCallback::init() {
+ int status = ML_ERROR_NONE;
+
+ status = DataBuffer::init();
+ if (status != ML_ERROR_NONE)
+ return status;
+
+ if (callback_train == nullptr)
+ return ML_ERROR_BAD_ADDRESS;
+
+ this->max_train = 0;
+ this->max_val = 0;
+ this->max_test = 0;
+
+ if (train_bufsize > batch_size || train_bufsize == 0) {
+ train_bufsize = batch_size;
+ }
+
+ if (val_bufsize > batch_size || val_bufsize == 0) {
+ val_bufsize = batch_size;
+ }
+
+ if (test_bufsize > batch_size || test_bufsize == 0) {
+ test_bufsize = batch_size;
+ }
+
+ this->train_running = true;
+ this->val_running = true;
+ this->test_running = true;
+
+ return ML_ERROR_NONE;
+}
+
+int DataBufferFromCallback::setGeneratorFunc(BufferType type, datagen_cb func) {
+
+ int status = ML_ERROR_NONE;
+ switch (type) {
+ case BufferType::BUF_TRAIN:
+ if (!func)
+ return ML_ERROR_INVALID_PARAMETER;
+ callback_train = func;
+ if (func)
+ validation[0] = true;
+ break;
+ case BufferType::BUF_VAL:
+ callback_val = func;
+ if (func)
+ validation[1] = true;
+ break;
+ case BufferType::BUF_TEST:
+ callback_test = func;
+ if (func)
+ validation[2] = true;
+ break;
+ default:
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+
+ return status;
+}
+
+void DataBufferFromCallback::updateData(BufferType type) {
+ int status = ML_ERROR_NONE;
+
+ unsigned int buf_size = 0;
+ unsigned int *cur_size = NULL;
+ bool *running = NULL;
+ std::vector<std::vector<float>> *data = NULL;
+ std::vector<std::vector<float>> *datalabel = NULL;
+ datagen_cb callback;
+
+ switch (type) {
+ case BufferType::BUF_TRAIN: {
+ buf_size = train_bufsize;
+ cur_size = &cur_train_bufsize;
+ running = &train_running;
+ data = &train_data;
+ datalabel = &train_data_label;
+ callback = callback_train;
+ } break;
+ case BufferType::BUF_VAL: {
+ buf_size = val_bufsize;
+ cur_size = &cur_val_bufsize;
+ running = &val_running;
+ data = &val_data;
+ datalabel = &val_data_label;
+ callback = callback_val;
+ } break;
+ case BufferType::BUF_TEST: {
+ buf_size = test_bufsize;
+ cur_size = &cur_test_bufsize;
+ running = &test_running;
+ data = &test_data;
+ datalabel = &test_data_label;
+ callback = callback_test;
+ } break;
+ default:
+ break;
+ }
+
+ try {
+ if ((cur_size == NULL) || (running == NULL) || (data == NULL) ||
+ (datalabel == NULL))
+ throw std::runtime_error("Error: assining error");
+ } catch (...) {
+ globalExceptionPtr = std::current_exception();
+ NN_EXCEPTION_NOTI(DATA_ERROR);
+ return;
+ }
+ bool endflag = false;
+
+ float **vec_arr = (float **)malloc(sizeof(float *) * 1);
+ float **veclabel_arr = (float **)malloc(sizeof(float *) * 1);
+
+ float *vec =
+ (float *)malloc(sizeof(float) * input_dim.batch() * input_dim.channel() *
+ input_dim.height() * input_dim.width());
+ float *veclabel =
+ (float *)malloc(sizeof(float) * input_dim.batch() * class_num);
+
+ vec_arr[0] = vec;
+ veclabel_arr[0] = veclabel;
+
+ while ((*running)) {
+ endflag = false;
+ NN_EXCEPTION_NOTI(DATA_NOT_READY);
+ if (buf_size - (*cur_size) > 0) {
+ /** @todo Update to support multiple inputs later */
+ status = callback(vec_arr, veclabel_arr, &endflag, user_data);
+ if (endflag) {
+ NN_EXCEPTION_NOTI(DATA_END);
+ free(vec);
+ free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
+ return;
+ }
+ if (status != ML_ERROR_NONE) {
+ NN_EXCEPTION_NOTI(DATA_ERROR);
+ free(vec);
+ free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
+ return;
+ }
+
+ for (unsigned int i = 0; i < input_dim.batch(); ++i) {
+ std::vector<float> v;
+ std::vector<float> vl;
+ unsigned int I =
+ i * input_dim.channel() * input_dim.height() * input_dim.width();
+ for (unsigned int j = 0; j < input_dim.channel(); ++j) {
+ unsigned int J = j * input_dim.height() * input_dim.width();
+ for (unsigned int k = 0; k < input_dim.height() * input_dim.width();
+ ++k) {
+ unsigned int K = I + J + k;
+ v.push_back(vec[K]);
+ }
+ }
+
+ I = i * class_num;
+ for (unsigned int j = 0; j < class_num; ++j) {
+ vl.push_back(veclabel[I + j]);
+ }
+
+ data_lock.lock();
+ data->push_back(v);
+ datalabel->push_back(vl);
+ (*cur_size)++;
+ data_lock.unlock();
+ }
+ }
+ if (buf_size == (*cur_size)) {
+ NN_EXCEPTION_NOTI(DATA_READY);
+ }
+ }
+
+ free(vec);
+ free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
+}
+
+int DataBufferFromCallback::setProperty(const PropertyType type,
+ std::string &value) {
+ return DataBuffer::setProperty(type, value);
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file databuffer_file.h
+ * @date 27 April 2020
+ * @brief This is buffer object take data from raw files
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __DATABUFFER_FUNC_H__
+#define __DATABUFFER_FUNC_H__
+#ifdef __cplusplus
+
+#include <functional>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include <databuffer.h>
+#include <nntrainer-api-common.h>
+
+namespace nntrainer {
+
+/**
+ * @class DataBufferFromCallback Data Buffer from callback given by user
+ * @brief Data Buffer from callback function
+ */
+class DataBufferFromCallback : public DataBuffer {
+public:
+ /**
+ * @brief Constructor
+ */
+ DataBufferFromCallback() : DataBuffer(DataBufferType::GENERATOR) {}
+
+ /**
+ * @brief Destructor
+ */
+ ~DataBufferFromCallback(){};
+
+ /**
+ * @brief Initialize Buffer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int init();
+
+ /**
+ * @brief set function pointer for each type
+ * @param[in] type Buffer Type
+ * @param[in] call back function pointer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setGeneratorFunc(BufferType type, datagen_cb func);
+
+ /**
+ * @brief Update Data Buffer ( it is for child thread )
+ * @param[in] BufferType training, validation, test
+ * @retval void
+ */
+ void updateData(BufferType type);
+
+ /**
+ * @brief set property
+ * @param[in] type type of property
+ * @param[in] value string value of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(const PropertyType type, std::string &value);
+
+private:
+ /**
+ *
+ * @brief Callback function to get user specific data
+ * @param[in] X data 3D float vector type
+ * @param[in] Y label 3D float vector type
+ * @param[out] status status for error handle
+ * @retval true / false generate all data for this epoch
+ *
+ */
+ datagen_cb callback_train;
+ datagen_cb callback_val;
+ datagen_cb callback_test;
+};
+} // namespace nntrainer
+#endif /* __cplusplus */
+#endif /* __DATABUFFER_FUNC_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file databuffer_util.h
+ * @date 12 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Databuffer utility file.
+ *
+ */
+
+#define SET_VALIDATION(val) \
+ do { \
+ for (DataType i = DATA_TRAIN; i < DATA_UNKNOWN; i = DataType(i + 1)) \
+ validation[i] = val; \
+ } while (0)
+
+#define NN_EXCEPTION_NOTI(val) \
+ do { \
+ switch (type) { \
+ case BufferType::BUF_TRAIN: { \
+ std::lock_guard<std::mutex> lgtrain(readyTrainData); \
+ trainReadyFlag = val; \
+ cv_train.notify_all(); \
+ } break; \
+ case BufferType::BUF_VAL: { \
+ std::lock_guard<std::mutex> lgval(readyValData); \
+ valReadyFlag = val; \
+ cv_val.notify_all(); \
+ } break; \
+ case BufferType::BUF_TEST: { \
+ std::lock_guard<std::mutex> lgtest(readyTestData); \
+ testReadyFlag = val; \
+ cv_test.notify_all(); \
+ } break; \
+ default: \
+ break; \
+ } \
+ } while (0)
--- /dev/null
+dataset_sources = [
+ 'databuffer.cpp',
+ 'databuffer_factory.cpp',
+ 'databuffer_file.cpp',
+ 'databuffer_func.cpp'
+]
+
+dataset_headers = [
+ 'databuffer.h',
+ 'databuffer_factory.h'
+]
+
+foreach s : dataset_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
+
+foreach h : dataset_headers
+ nntrainer_headers += join_paths(meson.current_source_dir(), h)
+endforeach
+
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0-only
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file delegate.h
+ * @date 7 Aug 2020
+ * @brief This is Delegate Class for the Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ * @note This class is experimental and subject to major modifications.
+ *
+ */
+
+#ifndef __DELEGATE_H__
+#define __DELEGATE_H__
+
+namespace nntrainer {
+
+/**
+ * @class Backend
+ * @brief Backend to be used for the operations to use.
+ * @note If some operation is not supported on set backend, that operation
+ * implemented with default backend will be used.
+ *
+ */
+class Backend {
+public:
+ /**
+ * @brief Backend type enumeration
+ */
+ enum class BackendType {
+ armcl, /**< Arm Compute Library for backend */
+ blas, /**< Libopenblas for backend */
+ base /**< Internally implemented operations for backend */
+ };
+
+ /**
+ * @brief Construct a new Backend object
+ *
+ * @param backend Backend of be used. Defaults to base backend
+ * @param num_t Number of threads. Defaults to 1 thread
+ */
+ Backend(BackendType backend = BackendType::base, int num_t = 1) :
+ backend(backend),
+ num_threads(num_t) {}
+
+ /**
+ * @brief Set the number of threads for the backend if supported
+ *
+ * @param num_threads Number of threads
+ */
+ void setNumThreads(unsigned int num_threads);
+
+private:
+ /**
+ * @brief backend type stored
+ */
+ BackendType backend;
+
+ /**
+ * @brief number of threads stored
+ */
+ unsigned int num_threads;
+};
+
+/**
+ * @class Device
+ * @brief Device to be used for the operations to run.
+ * @note The selected delegate device will restrict the supported backend.
+ *
+ */
+class Device {
+public:
+ /**
+ * @brief Device type enumeration
+ */
+ enum class DeviceType {
+ cpu, /**< CPU as the device */
+ gpu, /**< GPU as the device */
+ npu /**< NPU as the device */
+ };
+
+ /**
+ * @brief Construct a new Device object
+ *
+ * @param device Device delegate to be used for the operations to run. Default
+ * device is cpu.
+ * @param mem_frac Maximum memory fraction to be used of the set device.
+ * @param soft_place To enable soft device placement.
+ * @param prec_loss To enable precision loss for faster computation for the
+ * device.
+ */
+ Device(DeviceType device = DeviceType::cpu, float mem_frac = 1.0,
+ soft_place = false, prec_loss = false) :
+ device(device),
+ memory_fraction(mem_frac),
+ soft_placement(soft_place),
+ precision_loss(prec_loss) {}
+
+ /**
+ * @brief Set the Device object
+ *
+ * @param device The device to be set to
+ */
+ void setDevice(DeviceType device) { this->device = device; }
+
+ /**
+ * @brief Get the Device object
+ *
+ * @return DeviceType The device type which is set
+ */
+ DeviceType getDevice() { return device; }
+
+ /**
+ * @brief Set the maximum memory fraction which can be used by the framework
+ *
+ * @param memory_fraction Maximum fraction of memory to be used
+ */
+ void setMemoryFraction(float memory_fraction) {
+ throw std::logic_error("Not yet implemented");
+ }
+
+ /**
+ * @brief Allow placing the operations on device softly
+ * @details This allows framework to use some other device other than the set
+ * device for some of the operations for optimization or if an operation is
+ * not supported for the selected device. This might incur extra memory copy
+ * overhead.
+ * @note When set to false, constructing the model can result in error if a
+ * required operation is not avaiable for that device.
+ *
+ * @param soft_placement True to allow soft placement, else false
+ */
+ void allowSoftPlacement(bool soft_placement) {
+ throw std::logic_error("Not yet implemented");
+ this->soft_placement = soft_placement;
+ }
+
+ /**
+ * @brief Allow using low precision version of some of the operations
+ * @details This allows framework to use low precision operations (like
+ * float16 instead of float32) for some of the operations for higher
+ * performance with minimum impact to performance
+ *
+ * @param precision_loss True to allow loss in precision, else false
+ */
+ void allowPrecisionLoss(bool precision_loss) {
+ throw std::logic_error("Not yet implemented");
+ this->precision_loss = precision_loss;
+ }
+
+private:
+ DeviceType device; /**< Device type for the object */
+ float memory_fraction; /**< Maximum memory fraction which can be used by the
+ framework*/
+ bool soft_placement; /**< Allow placing the operations on device softly */
+ bool precision_loss; /**< Allow using low precision version of some of the
+ operations */
+};
+
+/**
+ * @class DelegateConfig
+ * @brief Configuration for the delegate
+ *
+ */
+class DelegateConfig {
+public:
+ /**
+ * @brief Construct a new Delegate Config object
+ *
+ * @param backend Backend to be set for this delegate
+ * @param device Device to be set for the delegate
+ */
+ DelegateConfig(Backend backend, Device device) :
+ backend(backend),
+ device(device) {}
+
+private:
+ Backend backend; /**< Backend to be used for the operations to use */
+ Device device; /**< Device to be used for the operations to run. */
+};
+
+} // namespace nntrainer
+
+#endif /* __DELEGATE_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jihoon Lee <jhoon.it.lee@samsung.com>
- *
- * @file activation_layer.h
- * @date 17 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jihoon Lee <jhoon.it.lee@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Activation Layer Class for Neural Network
- *
- */
-
-#ifndef __ACTIVATION_LAYER_H__
-#define __ACTIVATION_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class Activation Layer
- * @brief Activation Layer
- */
-class ActivationLayer : public Layer {
-
-public:
- /**
- * @brief Constructor of Activation Layer
- */
- template <typename... Args>
- ActivationLayer(ActivationType at = ActivationType::ACT_NONE, Args... args) :
- Layer(LayerType::LAYER_ACTIVATION, args...) {
- setActivation(at);
- }
-
- /**
- * @brief Destructor of Activation Layer
- */
- ~ActivationLayer(){};
-
- /**
- * @brief Initialize the layer
- *
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief Read Activation layer params. This is essentially noops for now.
- * @param[in] file input stream file
- */
- void read(std::ifstream &file){/* noop */};
-
- /**
- * @brief Save Activation layer params. This is essentially noops for now.
- * @param[in] file output stream file
- */
- void save(std::ofstream &file){/* noop */};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief setActivation by preset ActivationType
- *
- * @param[in] ActivationTypeeActivationTypeeActivationTypeet
- */
- void setActivation(ActivationType acti_type);
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Activation"; };
-
- /**
- * @brief Calculate softmax for Tensor Type
- * @param[in] t Tensor
- * @retval Tensor
- */
- static Tensor softmax(Tensor const &x);
-
- /**
- * @brief derivative softmax function for Tensor Type
- * @param[in] x Tensor
- * @retVal Tensor
- */
- static Tensor softmaxPrime(Tensor const &x,
- Tensor const &derivative = Tensor());
-
- /**
- * @brief sigmoid activation function
- * @param[in] x input
- */
- static float sigmoid(float x);
-
- /**
- * @brief derivative sigmoid function
- * @param[in] x input
- */
- static float sigmoidPrime(float x);
-
- /**
- * @brief tanh function for float type
- * @param[in] x input
- */
- static float tanhFloat(float x);
-
- /**
- * @brief derivative tanh function
- * @param[in] x input
- */
- static float tanhPrime(float x);
-
- /**
- * @brief relu activation function
- * @param[in] x input
- */
- static float relu(float x);
-
- /**
- * @brief derivative relu function
- * @param[in] x input
- */
- static float reluPrime(float x);
-
- /**
- * @brief no_op function
- * @param[in] x input
- */
- static float no_op(float x);
-
- /**
- * @brief no_op function
- * @param[in] x input
- */
- static float no_op_prime(float x);
-
-private:
- std::function<Tensor(Tensor const &)> _act_fn;
- std::function<Tensor(Tensor const &, Tensor const &)> _act_prime_fn;
-
- /**
- * @brief setActivation by custom activation function
- * @note apply derivative as this activation_prime_fn does not utilize
- * derivative
- * @param[in] std::function<Tensor(Tensor const &)> activation_fn activation
- * function to be used
- * @param[in] std::function<Tensor(Tensor const &)> activation_prime_fn
- * activation_prime_function to be used
- * @retval #ML_ERROR_NONE when successful
- */
- int setActivation(
- std::function<Tensor(Tensor const &)> const &activation_fn,
- std::function<Tensor(Tensor const &)> const &activation_prime_fn);
-
- /**
- * @brief setActivation by custom activation function
- * @note derivative not applied here as this activation_prime_fn applies
- * derivative itself
- * @param[in] std::function<Tensor(Tensor const &)> activation_fn activation
- * function to be used
- * @param[in] std::function<Tensor(Tensor const &, Tensor const &)>
- * activation_prime_fn activation_prime_function to be used
- * @retval #ML_ERROR_NONE when successful
- */
- int setActivation(std::function<Tensor(Tensor const &)> const &activation_fn,
- std::function<Tensor(Tensor const &, Tensor const &)> const
- &activation_prime_fn);
-
- /**
- * @brief setActivation by custom activation function
- * @note apply derivative as this activation_prime_fn does not utilize
- * derivative
- * @param[in] std::function<float(float const &)> activation_fn activation
- * function to be used
- * @param[in] std::function<float(float const &)> activation_prime_fn
- * activation_prime_function to be used
- * @retval #ML_ERROR_NONE when successful
- */
- int setActivation(
- std::function<float(float const)> const &activation_fn,
- std::function<float(float const)> const &activation_prime_fn);
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __ACTIVATION_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file adam.h
- * @date 6 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the Adam optimizer.
- */
-#ifndef __ADAM_H__
-#define __ADAM_H__
-#ifdef __cplusplus
-
-#include <optimizer_internal.h>
-
-namespace nntrainer {
-
-/**
- * @class Adam optimizer class
- * @brief Adam optimizer
- */
-class Adam : public Optimizer {
-public:
- /**
- * @brief Constructor of Optimizer Class
- */
- template <typename... Args>
- Adam(float lr = 0.001f, double b1 = 0.9f, double b2 = 0.999f,
- double ep = 1.0e-7f, Args... args) :
- Optimizer(OptType::ADAM, lr, args...),
- beta1(b1),
- beta2(b2),
- epsilon(ep) {}
-
- /**
- * @copydoc apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- * int iteration)
- */
- void apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- int iteration);
-
- /**
- * @brief get the base name for the optimizer
- * @retval base name of the optimizer
- */
- std::string getBaseName() { return "Adam"; };
-
- /**
- * @copydoc getLearningRate(int iteration)
- */
- double getLearningRate(int iteration);
-
- /**
- * @copydoc setProperty(const PropertyType type,
- const std::string &value = "")
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
- /**
- * @copydoc Optimizer::initialize(std::shared_ptr<Weight> params, unsigned int
- num_weights, bool setTensor)
- */
- int initialize(std::shared_ptr<Weight> params, unsigned int num_weights,
- bool setTensor);
-
- /**
- * @copydoc read(std::ifstream &file)
- */
- void read(std::ifstream &file);
-
- /**
- * @copydoc save(std::ofstream &file)
- */
- void save(std::ofstream &file);
-
- /**
- * @brief get beta1
- */
- double getBeta1() { return beta1; };
-
- /**
- * @brief get beta2
- */
- double getBeta2() { return beta2; };
-
- /**
- * @brief get epsilon
- */
- double getEpsilon() { return epsilon; }
-
-private:
- /**
- * @brief Internal Tensors for adam Optimizer
- */
- std::vector<std::pair<Tensor, Tensor>> weight_mv;
-
- double beta1; /** momentum for grad */
- double beta2; /** momentum for grad**2 */
- double epsilon; /** epsilon to protect overflow */
-};
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __ADAM_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file addition_layer.h
- * @date 30 July 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Addition Layer Class for Neural Network
- *
- */
-
-#ifndef __ADDITION_LAYER_H__
-#define __ADDITION_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class Addition Layer
- * @brief Addition Layer
- */
-class AdditionLayer : public Layer {
-public:
- /**
- * @brief Constructor of Addition Layer
- */
- template <typename... Args>
- AdditionLayer(unsigned int num_inputs_ = 0, Args... args) :
- Layer(LayerType::LAYER_ADDITION, args...) {
- num_inputs = num_inputs_;
- }
-
- /**
- * @brief Destructor of Addition Layer
- */
- ~AdditionLayer(){};
-
- /**
- * @brief Move constructor of AdditionLayer.
- * @param[in] AdditionLayer &&
- */
- AdditionLayer(AdditionLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs AdditionLayer to be moved.
- */
- AdditionLayer &operator=(AdditionLayer &&rhs) = default;
-
- /**
- * @brief initialize layer
- * @param[in] last last layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file){};
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &file){};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Addition"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __ADDITION_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file blas_interface.h
- * @date 28 Aug 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is dummy header for blas support
- *
- */
-
-#ifndef __BLAS_INTERFACE_H_
-#define __BLAS_INTERFACE_H_
-#ifdef __cplusplus
-
-#ifdef USE_BLAS
-extern "C" {
-#include <cblas.h>
-}
-#else
-enum CBLAS_ORDER { CblasRowMajor = 101, CblasColMajor = 102 };
-
-enum CBLAS_TRANSPOSE {
- CblasNoTrans = 111,
- CblasTrans = 112,
- CblasConjTrans = 113
-};
-#endif
-
-#ifdef USE_CUBLAS
-#include <helper_cuda.h>
-#include <helper_functions.h>
-#endif
-
-namespace nntrainer {
-
-/* TODO : need to scopy, sscal, snrm2 */
-void sscal(const int N, const float alpha, float *X, const int incX);
-
-float snrm2(const int N, const float *X, const int incX);
-
-void scopy(const unsigned int N, const float *X, const int incX, float *Y,
- const int intY);
-
-void saxpy(const unsigned int N, const float alpha, const float *X,
- const int incX, float *Y, const int incY);
-
-void sgemm(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, CBLAS_TRANSPOSE TransB,
- const unsigned int M, const unsigned int N, const unsigned int K,
- const float alpha, const float *A, const unsigned int lda,
- const float *B, const unsigned int ldb, const float beta, float *C,
- const unsigned int ldc);
-
-void sgemv(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, const unsigned int M,
- const unsigned int N, const float alpha, const float *A,
- const unsigned int lda, const float *X, const int incX,
- const float beta, float *Y, const int incY);
-
-} /* namespace nntrainer */
-#endif /* __cplusplus */
-#endif /* __BLAS_INTERFACE_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file bn_layer.h
- * @date 14 May 2020
- * @brief This is Batch Normalization Layer Class of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __BN_LAYER_H__
-#define __BN_LAYER_H__
-#ifdef __cplusplus
-
-#include <array>
-#include <functional>
-#include <vector>
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class BatchNormalizationLayer
- * @brief Batch Noramlization Layer
- */
-class BatchNormalizationLayer : public Layer {
-public:
- /**
- * @brief Constructor of Batch Noramlization Layer
- */
- template <typename... Args>
- BatchNormalizationLayer(
- int axis = -1, float momentum = 0.99, float epsilon = 0.001,
- WeightInitializer moving_mean_initializer = WeightInitializer::WEIGHT_ZEROS,
- WeightInitializer moving_variance_initializer =
- WeightInitializer::WEIGHT_ZEROS,
- WeightInitializer gamma_initializer = WeightInitializer::WEIGHT_ONES,
- WeightInitializer beta_initializer = WeightInitializer::WEIGHT_ONES,
- Args... args) :
- Layer(LayerType::LAYER_BN, args...),
- epsilon(epsilon),
- momentum(momentum),
- axis(axis),
- initializers{moving_variance_initializer, moving_variance_initializer,
- gamma_initializer, beta_initializer} {}
-
- /**
- * @brief Destructor of BatchNormalizationLayer
- */
- ~BatchNormalizationLayer() {}
-
- /**
- * @brief Move constructor of Pooling 2D Layer.
- * @param[in] BatchNormalization &&
- */
- BatchNormalizationLayer(BatchNormalizationLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs BatchNormalizationLayer to be moved.
- */
- BatchNormalizationLayer &operator=(BatchNormalizationLayer &&rhs) = default;
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief copy layer
- * @param[in] l layer to copy
- */
- void copy(std::shared_ptr<Layer> l);
-
- /**
- * @brief initialize layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "BatchNormalization"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- Tensor cvar; /**< training variance saved in bn_layer::forwarding and used in
- bn_layer::backwarding */
- Tensor invstd; /**< inversed training std for backward pass */
-
- Tensor deviation; /**< (input - current_average) */
-
- Tensor x_normalized; /**< normalized axis saved for backwarding */
- float epsilon; /**< epsilon */
- float momentum; /**< momentum */
- int axis; /**< Target axis, axis inferred at initialize when -1 */
-
- std::vector<unsigned int> axes_to_reduce; /**< target axes to reduce */
- std::array<WeightInitializer, 4> initializers; /**< weight initializers */
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __BN_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file concat_layer.h
- * @date 27 Oct 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Concat Layer Class for Neural Network
- *
- */
-
-#ifndef __CONCAT_LAYER_H__
-#define __CONCAT_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class Concat Layer
- * @brief Concat Layer
- */
-class ConcatLayer : public Layer {
-public:
- /**
- * @brief Constructor of Concat Layer
- */
- template <typename... Args>
- ConcatLayer(unsigned int num_inputs_ = 0, Args... args) :
- Layer(LayerType::LAYER_CONCAT, args...) {
- num_inputs = num_inputs_;
- }
-
- /**
- * @brief Destructor of Concat Layer
- */
- ~ConcatLayer(){};
-
- /**
- * @brief Move constructor of ConcatLayer.
- * @param[in] ConcatLayer &&
- */
- ConcatLayer(ConcatLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs ConcatLayer to be moved.
- */
- ConcatLayer &operator=(ConcatLayer &&rhs) = default;
-
- /**
- * @brief initialize layer
- * @param[in] last last layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file){};
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &file){};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Concat"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __CONCAT_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file conv2d_layer.h
- * @date 01 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Convolution Layer Class for Neural Network
- *
- */
-
-#ifndef __CONV2D_LAYER_H_
-#define __CONV2D_LAYER_H_
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-constexpr const unsigned int CONV2D_DIM = 2;
-
-/**
- * @class Convolution 2D Layer
- * @brief Convolution 2D Layer
- */
-class Conv2DLayer : public Layer {
-public:
- /**
- * @brief Constructor of Conv 2D Layer
- */
- template <typename... Args>
- Conv2DLayer(unsigned int filter_size_ = 0,
- const std::array<unsigned int, CONV2D_DIM> &kernel_size_ = {0, 0},
- const std::array<unsigned int, CONV2D_DIM> &stride_ = {1, 1},
- const std::array<unsigned int, CONV2D_DIM> &padding_ = {0, 0},
- bool normalization_ = false, bool standardization_ = false,
- Args... args) :
- Layer(LayerType::LAYER_CONV2D, args...),
- filter_size(filter_size_),
- kernel_size(kernel_size_),
- stride(stride_),
- padding(padding_),
- normalization(normalization_),
- standardization(standardization_) {}
-
- /**
- * @brief Destructor of Conv 2D Layer
- */
- ~Conv2DLayer() {}
-
- /**
- * @brief Move constructor of Conv 2D Layer.
- * @param[in] Conv2dLayer &&
- */
- Conv2DLayer(Conv2DLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs Conv2DLayer to be moved.
- */
- Conv2DLayer &operator=(Conv2DLayer &&rhs) = default;
-
- /**
- * @brief initialize layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file);
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &file);
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief copy layer
- * @param[in] l layer to copy
- */
- void copy(std::shared_ptr<Layer> l);
-
- /* TO DO : support keras type of padding */
- /* enum class PaddingType { */
- /* full = 0, */
- /* same = 1, */
- /* valid = 2, */
- /* unknown = 3, */
- /* }; */
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Convolution2D"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- unsigned int filter_size;
- std::array<unsigned int, CONV2D_DIM> kernel_size;
- std::array<unsigned int, CONV2D_DIM> stride;
- std::array<unsigned int, CONV2D_DIM> padding;
-
- bool normalization;
- bool standardization;
-
- /**
- * @brief set Parameter Size
- * @param[in] * size : size arrary
- * @param[in] type : Property type
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setSize(int *size, PropertyType type);
-
- /**
- * @brief set Parameter Size
- * @param[in] f number of filters
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setFilter(int f);
-
- /**
- * @brief set normalization
- * @param[in] enable boolean
- */
- void setNormalization(bool enable) { this->normalization = enable; };
-
- /**
- * @brief set standardization
- * @param[in] enable boolean
- */
- void setStandardization(bool enable) { this->standardization = enable; };
-
- /**
- * @brief calculation convolution
- * @param[in] in input tensor data
- * @param[in] indim input tensor dimension
- * @param[in] kernel convolution kernel data
- * @param[in] kdim convolution kernel dimension
- * @param[in] out output
- * @param[in] stride stride value : x, y direction
- * @param[in] bias bias data
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int conv2d(float *in, TensorDim indim, const float *kernel, TensorDim kdim,
- float *out, unsigned int const *stride, float bias);
-
- /**
- * @brief calculation convolution with cblas_*gemm
- * @param[in] mkernel kernel data
- * @param[in] kdim kernel data demension
- * @param[in] in input tensor
- * @param[in] outdim output tensor dimension
- * @param[in] stride stride value : x, y direction
- * @param[in] padd pad value : x, y direction
- * @param[out] out output data
- * @param[in] osize output size
- * @param[in] channel_mode loop with channel first
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int conv2d_gemm(const float *mkernel, TensorDim kdim, Tensor const &in,
- TensorDim outdim,
- const std::array<unsigned int, CONV2D_DIM> &stride,
- const std::array<unsigned int, CONV2D_DIM> &pad, float *out,
- unsigned int osize, bool channel_mode);
-
- /**
- * @brief reform the data to 2d matrix
- * @param[in] in_padded padded input data
- * @param[in] kdim kernel dimesion for define number of row
- * @param[out] inCol reformed data
- * @param[in] outdim output dimension
- * @param[in] mstride stride value : x, y direction
- * @param[in] channel_mode loop with channel first
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int im2col(Tensor in_padded, TensorDim kdim, float *inCol, TensorDim outdim,
- const std::array<unsigned int, CONV2D_DIM> &mstride,
- bool channel_mode);
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __CONV2D_LAYER_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer.h
- * @date 04 December 2019
- * @brief This is buffer object to handle big data
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __DATABUFFER_H__
-#define __DATABUFFER_H__
-#ifdef __cplusplus
-
-#include <memory>
-#include <random>
-#include <thread>
-#include <vector>
-
-#include <dataset.h>
-#include <tensor_dim.h>
-
-namespace nntrainer {
-
-/**
- * @brief Aliasing from ccapi ml::train
- */
-using DataBufferType = ml::train::DatasetType;
-using DatasetDataType = ml::train::DatasetDataType;
-using datagen_cb = ml::train::datagen_cb;
-
-/**
- * @brief Enumeration of data type
- */
-typedef enum {
- DATA_TRAIN =
- (int)ml::train::DatasetDataType::DATA_TRAIN, /** data for training */
- DATA_VAL =
- (int)ml::train::DatasetDataType::DATA_VAL, /** data for validation */
- DATA_TEST = (int)ml::train::DatasetDataType::DATA_TEST, /** data for test */
- DATA_LABEL = (int)ml::train::DatasetDataType::DATA_LABEL, /** label names */
- DATA_UNKNOWN =
- (int)ml::train::DatasetDataType::DATA_UNKNOWN /** data not known */
-} DataType;
-
-/**
- * @brief Enumeration of buffer type
- */
-enum class BufferType {
- BUF_TRAIN = DATA_TRAIN, /** BUF_TRAIN ( Buffer for training ) */
- BUF_VAL = DATA_VAL, /** BUF_VAL ( Buffer for validation ) */
- BUF_TEST = DATA_TEST, /** BUF_TEST ( Buffer for test ) */
- BUF_UNKNOWN = DATA_UNKNOWN /** BUF_UNKNOWN ( unknown ) */
-};
-
-/**
- * @class DataBuffer Data Buffers
- * @brief Data Buffer for read and manage data
- */
-class DataBuffer : public ml::train::Dataset {
-public:
- /**
- * @brief Create Buffer
- * @retval DataBuffer
- */
- DataBuffer(DataBufferType type);
-
- /**
- * @brief Initialize Buffer with data buffer private variables
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int init();
-
- /**
- * @brief Update Data Buffer ( it is for child thread )
- * @param[in] BufferType training, validation, test
- * @retval void
- */
- virtual void updateData(BufferType type) = 0;
-
- /**
- * @brief function for thread ( training, validation, test )
- * @param[in] BufferType training, validation, test
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int run(BufferType type);
-
- /**
- * @brief clear thread ( training, validation, test )
- * @param[in] BufferType training, validation, test
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int clear(BufferType type);
-
- /**
- * @brief clear all thread ( training, validation, test )
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int clear();
-
- /**
- * @brief get Data from Data Buffer using databuffer param
- * @param[in] BufferType training, validation, test
- * @param[out] out feature data ( batch_size size ), a contiguous and
- * allocated memory block should be passed
- * @param[out] label label data ( batch_size size ), a contiguous and
- * allocated memory block should be passed
- * @retval true/false
- */
- bool getDataFromBuffer(BufferType type, float *out, float *label);
-
- /**
- * @brief set number of class
- * @param[in] number of class
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setClassNum(unsigned int n);
-
- /**
- * @brief set buffer size
- * @param[in] buffer size
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setBufSize(unsigned int n);
-
- /**
- * @brief set batch size
- * @param[in] n batch size
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setBatchSize(unsigned int n);
-
- /**
- * @brief set feature size
- * @param[in] feature batch size. It is equal to input layer's hidden size
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setFeatureSize(TensorDim indim);
-
- /**
- * @brief set feature size
- * @retval max_train
- */
- unsigned int getMaxTrain() { return max_train; }
-
- /**
- * @brief set feature size
- * @retval max_val
- */
- unsigned int getMaxVal() { return max_val; }
-
- /**
- * @brief set feature size
- * @retval max_test
- */
- unsigned int getMaxTest() { return max_test; }
-
- /**
- * @brief Display Progress
- * @param[in] count calculated set ( batch_size size )
- * @param[in] type buffer type ( BUF_TRAIN, BUF_VAL, BUF_TEST )
- * @retval void
- */
- void displayProgress(const int count, BufferType type, float loss);
-
- /**
- * @brief return validation of data set
- * @retval validation
- */
- bool *getValidation() { return validation; }
-
- /**
- * @brief set property
- * @param[in] values values of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(std::vector<std::string> values);
-
- /**
- * @brief set property to allow setting user_data for cb
- * @param[in] values values of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(std::vector<void *> values);
-
- /**
- * @brief set function pointer for each type
- * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
- * @param[in] call back function pointer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setGeneratorFunc(DatasetDataType type, datagen_cb func) {
- return setGeneratorFunc((BufferType)type, func);
- }
-
- /**
- * @brief set function pointer for each type
- * @param[in] type Buffer Type
- * @param[in] call back function pointer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setGeneratorFunc(BufferType type, datagen_cb func);
-
- /**
- * @brief set train data file name
- * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
- * @param[in] path file path
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setDataFile(DatasetDataType type, std::string path) {
- return setDataFile((DataType)type, path);
- }
-
- /**
- * @brief set train data file name
- * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
- * @param[in] path file path
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setDataFile(DataType type, std::string path);
-
- enum class PropertyType {
- train_data = 0,
- val_data = 1,
- test_data = 2,
- label_data = 3,
- buffer_size = 4,
- unknown = 5
- };
-
-protected:
- /**
- * @brief Number of Data Set
- */
- static constexpr const unsigned int NBUFTYPE = 4;
-
- /**
- * @brief state of the data buffer while getting the data
- */
- typedef enum {
- DATA_NOT_READY = 0,
- DATA_READY = 1,
- DATA_END = 2,
- DATA_ERROR = 3,
- } DataStatus;
-
- /**
- * @brief status of thread
- */
- DataStatus trainReadyFlag;
- DataStatus valReadyFlag;
- DataStatus testReadyFlag;
-
- /**
- * @brief Data Queues for each data set
- */
- std::vector<std::vector<float>> train_data;
- std::vector<std::vector<float>> train_data_label;
- std::vector<std::vector<float>> val_data;
- std::vector<std::vector<float>> val_data_label;
- std::vector<std::vector<float>> test_data;
- std::vector<std::vector<float>> test_data_label;
-
- /**
- * @brief feature size
- */
- TensorDim input_dim;
-
- /**
- * @brief number of class
- */
- unsigned int class_num;
-
- /**
- * @brief number of remain data for each data queue
- */
- unsigned int cur_train_bufsize;
- unsigned int cur_val_bufsize;
- unsigned int cur_test_bufsize;
-
- /**
- * @brief queue size for each data set
- */
- unsigned int train_bufsize;
- unsigned int val_bufsize;
- unsigned int test_bufsize;
-
- unsigned int max_train;
- unsigned int max_val;
- unsigned int max_test;
-
- /**
- * @brief remain data set size
- */
- unsigned int rest_train;
- unsigned int rest_val;
- unsigned int rest_test;
-
- /**
- * @brief batch size
- */
- unsigned int batch_size;
-
- /**
- * @brief flags to check status
- */
- bool train_running;
- bool val_running;
- bool test_running;
-
- /**
- * @brief ids to check duplication
- */
- std::vector<unsigned int> train_mark;
- std::vector<unsigned int> val_mark;
- std::vector<unsigned int> test_mark;
-
- /**
- * @brief threads to generate data for queue
- */
- std::thread train_thread;
- std::thread val_thread;
- std::thread test_thread;
-
- std::vector<std::string> labels;
- bool validation[NBUFTYPE];
-
- /**
- * @brief return random int value between min to max
- * @param[in] min minimum vaule
- * @param[in] max maximum value
- * @retval int return value
- */
- int rangeRandom(int min, int max);
-
- std::mt19937 rng;
-
- /**
- * @brief The type of data buffer
- */
- DataBufferType data_buffer_type;
-
- /** The user_data to be used for the data generator callback */
- void *user_data;
-
- /**
- * @brief set property
- * @param[in] type type of property
- * @param[in] value string value of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setProperty(const DataBuffer::PropertyType type,
- std::string &value);
-};
-
-} // namespace nntrainer
-#endif /* __cplusplus */
-#endif /* __DATABUFFER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file databuffer_factory.h
- * @date 19 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the layer factory.
- */
-
-#ifndef __DATABUFFER_FACTORY_H__
-#define __DATABUFFER_FACTORY_H__
-#ifdef __cplusplus
-
-#include <databuffer.h>
-
-namespace nntrainer {
-
-/**
- * @brief Factory creator with constructor
- */
-std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type);
-
-/**
- * @brief Factory creator with constructor for databuffer with files
- */
-std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
- const char *train_file,
- const char *valid_file = nullptr,
- const char *test_file = nullptr);
-
-/**
- * @brief Factory creator with constructor for databuffer with callbacks
- */
-std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
- datagen_cb train,
- datagen_cb valid = nullptr,
- datagen_cb test = nullptr);
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __DATABUFFER_FACTORY_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.h
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __DATABUFFER_FILE_H__
-#define __DATABUFFER_FILE_H__
-#ifdef __cplusplus
-
-#include <fstream>
-#include <memory>
-#include <thread>
-#include <vector>
-
-#include <databuffer.h>
-
-namespace nntrainer {
-
-/**
- * @class DataBufferFromDataFile Data Buffer from Raw Data File
- * @brief Data Buffer from reading raw data
- */
-class DataBufferFromDataFile : public DataBuffer {
-
-public:
- /**
- * @brief Constructor
- */
- DataBufferFromDataFile() : DataBuffer(DataBufferType::FILE) {}
-
- /**
- * @brief Destructor
- */
- ~DataBufferFromDataFile() = default;
-
- /**
- * @brief Initialize Buffer with data buffer private variables
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int init();
-
- /**
- * @brief Update Data Buffer ( it is for child thread )
- * @param[in] BufferType training, validation, test
- * @retval void
- */
- void updateData(BufferType type);
-
- /**
- * @brief set train data file name
- * @param[in] type data type : DATA_TRAIN, DATA_VAL, DATA_TEST
- * @param[in] path file path
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setDataFile(DataType type, std::string path);
-
- /**
- * @brief set feature size
- * @param[in] Input Tensor Dimension.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setFeatureSize(TensorDim indim);
-
- /**
- * @brief set property
- * @param[in] type type of property
- * @param[in] value string value of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(const PropertyType type, std::string &value);
-
-private:
- /**
- * @brief raw data file names
- */
- std::string train_name;
- std::string val_name;
- std::string test_name;
-};
-
-} // namespace nntrainer
-#endif /* __cplusplus */
-#endif /* __DATABUFFER_FILE_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.h
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __DATABUFFER_FUNC_H__
-#define __DATABUFFER_FUNC_H__
-#ifdef __cplusplus
-
-#include <functional>
-#include <memory>
-#include <thread>
-#include <vector>
-
-#include <databuffer.h>
-#include <nntrainer-api-common.h>
-
-namespace nntrainer {
-
-/**
- * @class DataBufferFromCallback Data Buffer from callback given by user
- * @brief Data Buffer from callback function
- */
-class DataBufferFromCallback : public DataBuffer {
-public:
- /**
- * @brief Constructor
- */
- DataBufferFromCallback() : DataBuffer(DataBufferType::GENERATOR) {}
-
- /**
- * @brief Destructor
- */
- ~DataBufferFromCallback(){};
-
- /**
- * @brief Initialize Buffer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int init();
-
- /**
- * @brief set function pointer for each type
- * @param[in] type Buffer Type
- * @param[in] call back function pointer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setGeneratorFunc(BufferType type, datagen_cb func);
-
- /**
- * @brief Update Data Buffer ( it is for child thread )
- * @param[in] BufferType training, validation, test
- * @retval void
- */
- void updateData(BufferType type);
-
- /**
- * @brief set property
- * @param[in] type type of property
- * @param[in] value string value of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(const PropertyType type, std::string &value);
-
-private:
- /**
- *
- * @brief Callback function to get user specific data
- * @param[in] X data 3D float vector type
- * @param[in] Y label 3D float vector type
- * @param[out] status status for error handle
- * @retval true / false generate all data for this epoch
- *
- */
- datagen_cb callback_train;
- datagen_cb callback_val;
- datagen_cb callback_test;
-};
-} // namespace nntrainer
-#endif /* __cplusplus */
-#endif /* __DATABUFFER_FUNC_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file databuffer_util.h
- * @date 12 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Databuffer utility file.
- *
- */
-
-#define SET_VALIDATION(val) \
- do { \
- for (DataType i = DATA_TRAIN; i < DATA_UNKNOWN; i = DataType(i + 1)) \
- validation[i] = val; \
- } while (0)
-
-#define NN_EXCEPTION_NOTI(val) \
- do { \
- switch (type) { \
- case BufferType::BUF_TRAIN: { \
- std::lock_guard<std::mutex> lgtrain(readyTrainData); \
- trainReadyFlag = val; \
- cv_train.notify_all(); \
- } break; \
- case BufferType::BUF_VAL: { \
- std::lock_guard<std::mutex> lgval(readyValData); \
- valReadyFlag = val; \
- cv_val.notify_all(); \
- } break; \
- case BufferType::BUF_TEST: { \
- std::lock_guard<std::mutex> lgtest(readyTestData); \
- testReadyFlag = val; \
- cv_test.notify_all(); \
- } break; \
- default: \
- break; \
- } \
- } while (0)
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0-only
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file delegate.h
- * @date 7 Aug 2020
- * @brief This is Delegate Class for the Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- * @note This class is experimental and subject to major modifications.
- *
- */
-
-#ifndef __DELEGATE_H__
-#define __DELEGATE_H__
-
-namespace nntrainer {
-
-/**
- * @class Backend
- * @brief Backend to be used for the operations to use.
- * @note If some operation is not supported on set backend, that operation
- * implemented with default backend will be used.
- *
- */
-class Backend {
-public:
- /**
- * @brief Backend type enumeration
- */
- enum class BackendType {
- armcl, /**< Arm Compute Library for backend */
- blas, /**< Libopenblas for backend */
- base /**< Internally implemented operations for backend */
- };
-
- /**
- * @brief Construct a new Backend object
- *
- * @param backend Backend of be used. Defaults to base backend
- * @param num_t Number of threads. Defaults to 1 thread
- */
- Backend(BackendType backend = BackendType::base, int num_t = 1) :
- backend(backend),
- num_threads(num_t) {}
-
- /**
- * @brief Set the number of threads for the backend if supported
- *
- * @param num_threads Number of threads
- */
- void setNumThreads(unsigned int num_threads);
-
-private:
- /**
- * @brief backend type stored
- */
- BackendType backend;
-
- /**
- * @brief number of threads stored
- */
- unsigned int num_threads;
-};
-
-/**
- * @class Device
- * @brief Device to be used for the operations to run.
- * @note The selected delegate device will restrict the supported backend.
- *
- */
-class Device {
-public:
- /**
- * @brief Device type enumeration
- */
- enum class DeviceType {
- cpu, /**< CPU as the device */
- gpu, /**< GPU as the device */
- npu /**< NPU as the device */
- };
-
- /**
- * @brief Construct a new Device object
- *
- * @param device Device delegate to be used for the operations to run. Default
- * device is cpu.
- * @param mem_frac Maximum memory fraction to be used of the set device.
- * @param soft_place To enable soft device placement.
- * @param prec_loss To enable precision loss for faster computation for the
- * device.
- */
- Device(DeviceType device = DeviceType::cpu, float mem_frac = 1.0,
- soft_place = false, prec_loss = false) :
- device(device),
- memory_fraction(mem_frac),
- soft_placement(soft_place),
- precision_loss(prec_loss) {}
-
- /**
- * @brief Set the Device object
- *
- * @param device The device to be set to
- */
- void setDevice(DeviceType device) { this->device = device; }
-
- /**
- * @brief Get the Device object
- *
- * @return DeviceType The device type which is set
- */
- DeviceType getDevice() { return device; }
-
- /**
- * @brief Set the maximum memory fraction which can be used by the framework
- *
- * @param memory_fraction Maximum fraction of memory to be used
- */
- void setMemoryFraction(float memory_fraction) {
- throw std::logic_error("Not yet implemented");
- }
-
- /**
- * @brief Allow placing the operations on device softly
- * @details This allows framework to use some other device other than the set
- * device for some of the operations for optimization or if an operation is
- * not supported for the selected device. This might incur extra memory copy
- * overhead.
- * @note When set to false, constructing the model can result in error if a
- * required operation is not avaiable for that device.
- *
- * @param soft_placement True to allow soft placement, else false
- */
- void allowSoftPlacement(bool soft_placement) {
- throw std::logic_error("Not yet implemented");
- this->soft_placement = soft_placement;
- }
-
- /**
- * @brief Allow using low precision version of some of the operations
- * @details This allows framework to use low precision operations (like
- * float16 instead of float32) for some of the operations for higher
- * performance with minimum impact to performance
- *
- * @param precision_loss True to allow loss in precision, else false
- */
- void allowPrecisionLoss(bool precision_loss) {
- throw std::logic_error("Not yet implemented");
- this->precision_loss = precision_loss;
- }
-
-private:
- DeviceType device; /**< Device type for the object */
- float memory_fraction; /**< Maximum memory fraction which can be used by the
- framework*/
- bool soft_placement; /**< Allow placing the operations on device softly */
- bool precision_loss; /**< Allow using low precision version of some of the
- operations */
-};
-
-/**
- * @class DelegateConfig
- * @brief Configuration for the delegate
- *
- */
-class DelegateConfig {
-public:
- /**
- * @brief Construct a new Delegate Config object
- *
- * @param backend Backend to be set for this delegate
- * @param device Device to be set for the delegate
- */
- DelegateConfig(Backend backend, Device device) :
- backend(backend),
- device(device) {}
-
-private:
- Backend backend; /**< Backend to be used for the operations to use */
- Device device; /**< Device to be used for the operations to run. */
-};
-
-} // namespace nntrainer
-
-#endif /* __DELEGATE_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file fc_layer.h
- * @date 14 May 2020
- * @brief This is Fully Connected Layer Class of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __FC_LAYER_H__
-#define __FC_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class FullyConnecedLayer
- * @brief fully connected layer
- */
-class FullyConnectedLayer : public Layer {
-public:
- /**
- * @brief Constructor of Fully Connected Layer
- */
- template <typename... Args>
- FullyConnectedLayer(unsigned int unit_ = 0, Args... args) :
- Layer(LayerType::LAYER_FC, args...),
- unit(unit_) {}
-
- /**
- * @brief Destructor of Fully Connected Layer
- */
- ~FullyConnectedLayer(){};
-
- /**
- * @brief Move constructor.
- * @param[in] FullyConnected &&
- */
- FullyConnectedLayer(FullyConnectedLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs FullyConnectedLayer to be moved.
- */
- FullyConnectedLayer &operator=(FullyConnectedLayer &&rhs) = default;
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file);
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &file);
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief copy layer
- * @param[in] l layer to copy
- */
- void copy(std::shared_ptr<Layer> l);
-
- /**
- * @brief initialize layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "FullyConnected"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- unsigned int unit;
-};
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __FC_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file flatten_layer.h
- * @date 16 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Flatten Layer Class for Neural Network
- *
- */
-
-#ifndef __FLATTEN_LAYER_H__
-#define __FLATTEN_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class Flatten Layer
- * @brief Flatten Layer
- */
-class FlattenLayer : public Layer {
-public:
- /**
- * @brief Constructor of Flatten Layer
- */
- template <typename... Args>
- FlattenLayer(Args... args) : Layer(LayerType::LAYER_FLATTEN, args...) {}
-
- /**
- * @brief Destructor of Flatten Layer
- */
- ~FlattenLayer(){};
-
- /**
- * @brief Move constructor of FlattenLayer.
- * @param[in] FlattenLayer &&
- */
- FlattenLayer(FlattenLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs FlattenLayer to be moved.
- */
- FlattenLayer &operator=(FlattenLayer &&rhs) = default;
-
- /**
- * @brief initialize layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file){};
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &file){};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Flatten"; };
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __FLATTEN_LAYER_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file input_layer.h
- * @date 14 May 2020
- * @brief This is Input Layer Class of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __INPUT_LAYER_H__
-#define __INPUT_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class Input Layer
- * @brief Just Handle the Input of Network
- */
-class InputLayer : public Layer {
-public:
- /**
- * @brief Constructor of InputLayer
- */
- template <typename... Args>
- InputLayer(bool normalization = false, bool standardization = false,
- Args... args) :
- Layer(LayerType::LAYER_IN, args...),
- normalization(false),
- standardization(false) {}
-
- /**
- * @brief Destructor of InputLayer
- */
- ~InputLayer() {}
-
- /**
- * @brief Move constructor of Pooling 2D Layer.
- * @param[in] Input &&
- */
- InputLayer(InputLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs InputLayer to be moved.
- */
- InputLayer &operator=(InputLayer &&rhs) = default;
-
- /**
- * @brief No Weight data for this Input Layer
- */
- void read(std::ifstream &file){};
-
- /**
- * @brief No Weight data for this Input Layer
- */
- void save(std::ofstream &file){};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief Initializer of Input Layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Input"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- bool normalization;
- bool standardization;
-
- /**
- * @brief set normalization
- * @param[in] enable boolean
- */
- void setNormalization(bool enable) { this->normalization = enable; };
-
- /**
- * @brief set standardization
- * @param[in] enable boolean
- */
- void setStandardization(bool enable) { this->standardization = enable; };
-};
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __INPUT_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file layer_factory.h
- * @date 7 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the layer factory.
- */
-
-#ifndef __LAYER_FACTORY_H__
-#define __LAYER_FACTORY_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-
-namespace nntrainer {
-
-/**
- * @brief Factory creator with constructor
- */
-std::unique_ptr<Layer> createLayer(LayerType type);
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __LAYER_FACTORY_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file layer_internal.h
- * @date 04 December 2019
- * @brief This is Layer classes of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-#ifndef __LAYER_H__
-#define __LAYER_H__
-#ifdef __cplusplus
-
-#include <memory>
-#include <set>
-#include <vector>
-
-#include <layer.h>
-#include <optimizer_internal.h>
-#include <tensor.h>
-#include <tensor_dim.h>
-#include <weight.h>
-
-namespace nntrainer {
-
-/**
- * @brief Enumeration of activation function type
- */
-enum class ActivationType {
- ACT_TANH, /** tanh */
- ACT_SIGMOID, /** sigmoid */
- ACT_RELU, /** ReLU */
- ACT_SOFTMAX, /** softmax */
- ACT_NONE, /** no op */
- ACT_UNKNOWN /** unknown */
-};
-
-using LayerType = ml::train::LayerType;
-
-/**
- * @class Layer Base class for layers
- * @brief Base class for all layers
- */
-class Layer : public ml::train::Layer {
-
- /** model classes can call private methods which arent exposed to public */
- friend class NeuralNetwork;
- friend class ModelLoader;
-
-public:
- /**
- * @brief Constructor of Layer Class
- */
- Layer(
- LayerType type_, ActivationType activation_type_ = ActivationType::ACT_NONE,
- WeightRegularizerType weight_regularizer_ = WeightRegularizerType::unknown,
- const float weight_regularizer_constant_ = 1.0f,
- WeightInitializer weight_initializer_ =
- WeightInitializer::WEIGHT_XAVIER_UNIFORM,
- WeightInitializer bias_initializer_ = WeightInitializer::WEIGHT_ZEROS,
- bool trainable_ = true, bool flatten_ = false) :
- name(std::string()),
- type(type_),
- loss(0.0f),
- activation_type(activation_type_),
- weight_regularizer(weight_regularizer_),
- weight_regularizer_constant(weight_regularizer_constant_),
- weight_initializer(weight_initializer_),
- bias_initializer(bias_initializer_),
- flatten(flatten_),
- trainable(trainable_),
- num_weights(0),
- num_inputs(1),
- num_outputs(1) {
- input_dim.resize(1);
- output_dim.resize(1);
- }
-
- /**
- * @brief Move constructor of Layer.
- * @param[in] Layer &&
- */
- Layer(Layer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs Layer to be moved.
- */
- virtual Layer &operator=(Layer &&rhs) = default;
-
- /**
- * @brief Forward Propagation of a layer
- * @param[in] in List of Input Tensors taken by this layer
- * @retval List of Output Tensors
- */
- virtual sharedConstTensors forwarding(sharedConstTensors in) = 0;
-
- /**
- * @brief Back Propagation of a layer
- * @param[in] in List of Derivative Tensor from the next layer
- * @param[in] iteration Iteration value for the Optimizer
- * @retval Derivative List of Tensor for the previous layer
- */
- virtual sharedConstTensors backwarding(sharedConstTensors in,
- int iteration) = 0;
-
- /**
- * @brief read layer Weight & Bias data from file
- * @note derived class can call this to get/save weights
- * @param[in] file input file stream
- */
- virtual void read(std::ifstream &file);
-
- /**
- * @brief save layer Weight & Bias data from file
- * @note derived class can call this to get/save weights
- * @param[in] file output file stream
- */
- virtual void save(std::ofstream &file);
-
- /**
- * @brief set Property of layer
- * @param[in] values values of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(std::vector<std::string> values);
-
- /**
- * @brief setProperty by PropertyType
- * @note By passing empty string, this can validate if @a type is valid
- * @param[in] type property type to be passed
- * @param[in] value value to be passed, if empty string is passed, do nothing
- * but throws error when @a type is invalid
- * @exception exception::not_supported when property type is not valid for
- * the particular layer
- * @exception std::invalid_argument invalid argument
- */
- virtual void setProperty(const PropertyType type,
- const std::string &value = "");
-
- /**
- * @brief Optimizer Setter
- * @param[in] opt Optimizer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setOptimizer(std::shared_ptr<Optimizer> opt);
-
- /**
- * @brief Get the Optimizer object
- *
- * @return std::shared_ptr<Optimizer> optimizer
- */
- std::shared_ptr<Optimizer> getOptimizer() { return opt; }
-
- /**
- * @brief Activation Type Getter
- * @retval Activation Type.
- */
- ActivationType getActivationType() { return this->activation_type; }
-
- /**
- * @brief Layer type Getter
- * @retval type LayerType
- */
- LayerType getType() { return type; }
-
- /**
- * @brief Copy Layer
- * @param[in] l Layer to be copied
- */
- virtual void copy(std::shared_ptr<Layer> l);
-
- /**
- * @brief check hyper parameter for the layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int checkValidation();
-
- /**
- * @brief Get the output dimension
- * @return TensorDim dimension of the output
- */
- std::vector<TensorDim> getOutputDimension() { return output_dim; }
-
- /**
- * @brief Get the input dimension
- * @return TensorDim dimension of the input
- */
- std::vector<TensorDim> getInputDimension() { return input_dim; }
-
- /**
- * @brief get the loss value added by this layer
- * @retval loss value
- */
- float getLoss() { return loss; }
-
- /**
- * @brief set trainable for this layer
- * @param[in] train to enable/disable train
- */
- virtual void setTrainable(bool train) { trainable = train; }
-
- /**
- * @brief get trainable for this layer
- * @retval train to enable/disable train
- */
- bool getTrainable() noexcept { return trainable; }
-
- /**
- * @brief get all weights of the layer
- * @retval vector of all params
- */
- std::shared_ptr<Weight> getWeights() { return weight_list; }
-
- /**
- * @brief get if the output of this layer must be flatten
- * @retval flatten value
- */
- bool getFlatten() { return flatten; }
-
- /**
- * @brief Set name of the layer
- */
- int setName(std::string name);
-
- /**
- * @brief Get name of the layer
- */
- std::string getName() noexcept { return name; }
-
- /**
- * @brief print using PrintPreset
- *
- * @param out oustream
- * @param preset preset to be used
- */
- void printPreset(std::ostream &out,
- PrintPreset preset = PrintPreset::PRINT_SUMMARY);
-
- /**
- * @brief get data alias at param position.
- * @exception std::out_of_range for index out of range
- */
- Weight &weightAt(const unsigned int position) {
- if (position >= num_weights) {
- throw std::out_of_range("index out of range");
- }
-
- return weight_list.get()[position];
- }
-
- /**
- * @brief Get the number of weights
- *
- * @return unsigned int number of weights
- */
- unsigned int getNumWeights() { return num_weights; }
-
- /**
- * @brief Set the batch for the layer
- * @param batch Batch value to be set
- * @todo Make this private. Only model should be able to do this.
- */
- virtual void setBatch(unsigned int batch);
-
-protected:
- /**
- * @brief Print Options when printing layer info
- */
- typedef enum {
- // clang-format off
- PRINT_INST_INFO = (1 << 0), /**< Option to print type & instance address info */
- PRINT_SHAPE_INFO = (1 << 1), /**< Option to print shape information, invalid before initiation*/
- PRINT_PROP = (1 << 2), /**< Option to print properties */
- PRINT_PROP_META = (1 << 3), /**< Option to print properties that describe meta info
- e.g) layer activation type for non-activation layer. */
- PRINT_WEIGHTS = (1 << 4), /**< Option to print weights */
- PRINT_METRIC = (1 << 5) /**< Option to print metrics (currently loss only) */
- // clang-format on
- } PrintOption;
-
- /**
- * @brief Name of the layer (works as the identifier)
- */
- std::string name;
-
- /**
- * @brief check if current layer's weight decay type is l2norm
- * @return bool is weightdecay type is L2 Norm
- */
- bool isWeightRegularizerL2Norm() {
- return weight_regularizer == WeightRegularizerType::l2norm;
- }
- /**
- * @brief Input Tensor
- */
- Tensor input;
-
- /**
- * @brief Hidden Layer Tensor which store the
- * forwading result
- */
- Tensor hidden;
-
- /**
- * @brief Dimension of input activation
- */
- std::vector<TensorDim> input_dim;
-
- /**
- * @brief Dimension of output activation
- */
- std::vector<TensorDim> output_dim;
-
- /**
- * @brief Optimizer for this layer
- */
- // TODO: fix with #630
- std::shared_ptr<Optimizer> opt;
-
- /**
- * @brief Layer type
- */
- LayerType type;
-
- /**
- * @brief Loss value added by this layer
- */
- float loss;
-
- ActivationType activation_type;
-
- WeightRegularizerType weight_regularizer;
-
- float weight_regularizer_constant;
-
- WeightInitializer weight_initializer; /** initializer for weights */
-
- WeightInitializer bias_initializer; /** initializer for bias */
-
- /**
- * @brief Output of this layer should be flattened
- */
- bool flatten;
-
- /*
- * @brief making this false will skip updating this layer variables
- */
- bool trainable;
-
- /**
- * @brief reserve memory for @a weight_list and set @a num_weights
- * @exception std::invalid_argument when num_weights is already set and
- * shouldn't be changed again.
- */
- void setNumWeights(unsigned int psize) {
- if (psize == num_weights)
- return;
-
- if (num_weights > 0) {
- throw std::invalid_argument("param size can't be set once it is set");
- }
-
- num_weights = psize;
- weight_list = std::shared_ptr<Weight>(new Weight[num_weights],
- std::default_delete<Weight[]>());
- }
-
- /**
- * @brief weight_list in this layer. This contains trainable weights of
- * layers.
- */
- std::shared_ptr<Weight> weight_list;
-
- unsigned int num_weights; /**< length of weights.
- This shouldn't be changed
- after initiation
- use setNumWeights() to avoid
- setting parameters twice */
-
- /**
- * @brief Number of inputs this layer will requries/will operate on
- */
- unsigned int num_inputs;
-
- /**
- * @brief Numer of outputs this layer will produce
- */
- unsigned int num_outputs;
-
- /**
- * @brief Layer type Setter
- * @param[in] type layer type
- */
- void setType(LayerType type) { this->type = type; }
-
- /**
- * @brief Activation Setter
- * @param[in] activation activation type
- * @throw std::invalid_argument when ActivationType is unknown
- */
- virtual void setActivation(ActivationType activation);
-
-private:
- /**
- * @brief Set containing all the names of layers
- */
- static std::set<std::string> layer_names;
-
- /**
- * @brief Count assigned to layer names declared by default
- */
- static int def_name_count;
-
- /**
- * @brief Ensure that layer has a name
- */
- void ensureName();
-
- /**
- * @brief check if @a type is valid and print if prop is valid to @a out
- */
- template <typename T>
- void printIfValid(std::ostream &out, const PropertyType type, T target);
-
- /**
- * @brief anchor point to override if PRINT_SHAPE_INFO is enabled for
- * Layer::print()
- */
- virtual void printShapeInfo(std::ostream &out);
-
- /**
- * @brief anchor point to override if PRINT_PROP_META is enabled for
- * Layer::print()
- */
- virtual void printPropertiesMeta(std::ostream &out);
-
- /**
- * @brief anchor point to override if PRINT_PROP is enabled for Layer::print()
- */
- virtual void printProperties(std::ostream &out);
-
- /**
- * @brief anchor point to override if PRINT_METRIC is enabled for
- * Layer::print()
- */
- virtual void printMetric(std::ostream &out);
-
- /**
- * @brief set weight decay parameters
- * @param[in] w struct for weight decay
- */
- void setWeightRegularizer(WeightRegularizerType type) {
- weight_regularizer = type;
- }
-
- /**
- * @brief set Weight Initialization Type
- * @param[in] wini WeightInitializer
- */
- void setWeightInit(WeightInitializer wini) { weight_initializer = wini; }
-
- /**
- * @brief get if the output of this layer must be flatten
- * @retval flatten value
- */
- void setFlatten(bool flatten) { this->flatten = flatten; }
-
- /**
- * @brief Print layer related information. Do not override without clear
- * reason. It is recommended to override printShapeInfo, printPropertiesMeta,
- * printProperties, printMetric instead
- * @param[in] out outstream
- * @param[in] flags combination of LayerPrintOption
- */
- virtual void print(std::ostream &out, unsigned int flags = 0);
-
- /**
- * @brief Get base name of the layer
- */
- virtual std::string getBaseName() = 0;
-
- /**
- * @brief Initialize the layer
- * - Weight(Height, Width), Bias(1, Width)
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int initialize() = 0;
-
- /**
- * @brief Set the input dimension
- * @param[in] d dimension to be set
- */
- void setInputDimension(std::vector<TensorDim> d) { input_dim = d; }
-};
-
-/**
- * @brief Overriding output stream for layers and it's derived class
- */
-template <typename T, typename std::enable_if_t<
- std::is_base_of<Layer, T>::value, T> * = nullptr>
-std::ostream &operator<<(std::ostream &out, T &l) {
- l.printPreset(out, Layer::PrintPreset::PRINT_SUMMARY);
- return out;
-}
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/* Copyright (C) 2020 Jihoon Lee <jihoon.it.lee@samsung.com>
- *
- * @file lazy_tensor.h
- * @date 05 Jun 2020
- * @brief A lazy evaluation calculator for tensors
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jihoon Lee <jihoon.it.lee@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __LAZY_TENSOR_H__
-#define __LAZY_TENSOR_H__
-#ifdef __cplusplus
-
-#include <tensor.h>
-#include <vector>
-
-namespace nntrainer {
-
-/**
- * @class LazyTensor a wrapper class for lazy calculation of tensor
- * @brief calculation is delayed until Tensor LazyTensor::run() is
- * called, can be contructed by Tensor::chain() method
- */
-class LazyTensor {
-public:
- /**
- * @brief Constructor of Lazy Tensor, Tensor is copied to gaurantee
- * immutability
- */
- LazyTensor(const Tensor &from) { target.copy(from); };
-
- /**
- * @brief Wrapper method of add_i. see tensor.h for more detail
- * @param[in] value to be added
- * @retval LazyTensor *this
- */
- LazyTensor &add_i(float const &value);
-
- /**
- * @brief Wrapper method of add_i. see tensor.h for more detail
- * @param[in] m Tensor to be added
- * @retval LazyTensor *this
- */
- LazyTensor &add_i(Tensor const &m, float const alpha = 1);
-
- /**
- * @brief Wrapper method of subtract_i. see tensor.h for more detail
- * @param[in] m Tensor to subtract
- * @retval LazyTensor *this
- */
- LazyTensor &subtract_i(Tensor const &m);
-
- /**
- * @brief Wrapper method of subtract_i. see tensor.h for more detail
- * @param[in] value value to subtract
- * @retval LazyTensor *this
- */
- LazyTensor &subtract_i(float const &value);
-
- /**
- * @brief Wrapper method of multiply_i. see tensor.h for more detail
- * @param[in] value to be added
- * @retval LazyTensor *this
- */
- LazyTensor &multiply_i(float const &value);
-
- /**
- * @brief Wrapper method of multiply_i. see tensor.h for more detail
- * @param[in] m Tensor to be multiplied
- * @retval LazyTensor *this
- */
- LazyTensor &multiply_i(Tensor const &m);
-
- /**
- * @brief Wrapper method of divide_i. see tensor.h for more detail
- * @param[in] value divisor
- * @retval LazyTensor *this
- */
- LazyTensor ÷_i(float const &value);
-
- /**
- * @brief Wrapper method of divide_i. see tensor.h for more detail
- * @param[in] m Tensor to for division
- * @retval LazyTensor *this
- */
- LazyTensor ÷_i(Tensor const &m);
-
- /**
- * @brief Wrapper method of dot. see tensor.h for more detail (memcopy
- * happens)
- * @param[in] m Tensor
- * @retval LazyTensor *this
- */
- LazyTensor &dot(Tensor const &m);
-
- /**
- * @brief Wrapper method of transpose. see tensor.h for more detail
- * (memcopy happens)
- * @param[in] direction to transpose ex) 0:2:1
- * @retval LazyTensor *this
- */
- LazyTensor &transpose(std::string direction);
-
- /**
- * @brief Wrapper method of sum_by_batch. see tensor.h for more detail
- * (memcopy happens)
- * @retval LazyTensor *this
- */
- LazyTensor &sum_by_batch();
-
- /**
- * @brief Wrapper method of sum. see tensor.h for more detail (memcopy
- * happens) 0 : batch direction 1 : channel direction 2 : height direction 3 :
- * width direction
- * @retval LazyTensor *this
- */
- LazyTensor &sum(int axis);
-
- /**
- * @brief Wrapper method of average. see tensor.h for more detail (memcopy
- * happens) 0 : batch direction 1 : channel direction 2 : height direction 3 :
- * width direction
- * @retval LazyTensor *this
- */
- LazyTensor &average(int axis);
-
- /**
- * @brief Wrapper method of average. see tensor.h for more detail (memcopy
- * happens)
- * @retval LazyTensor *this
- */
- LazyTensor &average();
-
- /**
- * @brief execute the call_chain to get the tensor
- * @retval calculated tensor
- */
- Tensor run();
-
-private:
- /**< handle the data as a std::vector type */
- std::vector<std::function<int(Tensor &)>> call_chain;
- Tensor target;
-};
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __LAZY_TENSOR_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file loss_layer.h
- * @date 12 June 2020
- * @brief This is Loss Layer Class of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __LOSS_LAYER_H__
-#define __LOSS_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @brief Enumeration of loss function type
- */
-enum class LossType {
- LOSS_MSE, /** Mean Squared Error */
- LOSS_ENTROPY, /** Cross Entropy */
- LOSS_ENTROPY_SIGMOID, /** Cross Entropy amalgamated with sigmoid for stability
- */
- LOSS_ENTROPY_SOFTMAX, /** Cross Entropy amalgamated with softmax for stability
- */
- LOSS_UNKNOWN /** Unknown */
-};
-
-/**
- * @class LossLayer
- * @brief loss layer
- */
-class LossLayer : public Layer {
-public:
- /**
- * @brief Constructor of Loss Layer
- */
- template <typename... Args>
- LossLayer(LossType loss_type_ = LossType::LOSS_UNKNOWN, Args... args) :
- Layer(LayerType::LAYER_LOSS, args...),
- loss_type(LossType::LOSS_UNKNOWN) {}
-
- /**
- * @brief Destructor of Loss Layer
- */
- ~LossLayer(){};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @brief Forward Propagation of a layer
- * @param[in] in List of Input Tensors taken by this layer
- * @param[in] label List of Label Tensors for the model
- * @retval List of Input Tensors as it is.
- */
- sharedConstTensors forwarding(sharedConstTensors in,
- sharedConstTensors label);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @brief read layer Weight & Bias data from file
- * @param[in] file input file stream
- */
- void read(std::ifstream &file) {}
-
- /**
- * @brief save layer Weight & Bias data from file
- * @param[in] file output file stream
- */
- void save(std::ofstream &file) {}
-
- /**
- * @brief copy layer
- * @param[in] l layer to copy
- */
- void copy(std::shared_ptr<Layer> l);
-
- /**
- * @brief initialize layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Loss"; };
-
- /**
- * @brief set loss function
- * @param[in] l loss type
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setLoss(LossType l);
-
-private:
- LossType loss_type; /**< loss type of loss layer */
-
- /**
- * @brief update loss
- * @param[in] l Tensor data to calculate
- */
- void updateLoss(const Tensor &l);
-};
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __LOSS_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file model_loader.h
- * @date 5 August 2020
- * @brief This is model loader class for the Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __MODEL_LOADER_H__
-#define __MODEL_LOADER_H__
-#ifdef __cplusplus
-
-#include <iniparser.h>
-#include <neuralnet.h>
-
-namespace nntrainer {
-
-/**
- * @class ModelLoader
- * @brief Model Loader class to load model from various config files
- */
-class ModelLoader {
-public:
- /**
- * @brief Constructor of the model loader
- */
- ModelLoader() {}
-
- /**
- * @brief Destructor of the model loader
- */
- ~ModelLoader() {}
-
- /**
- * @brief load all of model and dataset from given config file
- * @param[in] config config file path
- * @param[in/out] model model to be loaded
- */
- int loadFromConfig(std::string config, NeuralNetwork &model);
-
-private:
- /**
- * @brief load all of model from given config file
- * @param[in] config config file path
- * @param[in/out] model model to be loaded
- * @param[in] bare_layers load only the layers as backbone if enabled
- */
- int loadFromConfig(std::string config, NeuralNetwork &model,
- bool bare_layers);
-
- /**
- * @brief load all of model and dataset from ini
- * @param[in] ini_file config file path
- * @param[in/out] model model to be loaded
- */
- int loadFromIni(std::string ini_file, NeuralNetwork &model, bool bare_layers);
-
- /**
- * @brief load dataset config from ini
- * @param[in] ini dictionary containing the config
- * @param[in] model model to be loaded
- */
- int loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model);
-
- /**
- * @brief load model config from ini
- * @param[in] ini dictionary containing the config
- * @param[in/out] model model to be loaded
- */
- int loadModelConfigIni(dictionary *ini, NeuralNetwork &model);
-
- /**
- * @brief load layer config from ini given the layer type
- * @param[in] ini dictionary containing the config
- * @param[in/out] layer layer to be loaded
- * @param[in] layer_name name of the layer to be loaded
- * @param[in] layer_type type of the layer to be loaded
- */
- int loadLayerConfigIniCommon(dictionary *ini, std::shared_ptr<Layer> &layer,
- const std::string &layer_name,
- LayerType layer_type);
-
- /**
- * @brief wrapper function to load layer config from ini
- * @param[in] ini dictionary containing the config
- * @param[in/out] layer layer to be loaded
- * @param[in] layer_name name of the layer to be loaded
- */
- int loadLayerConfigIni(dictionary *ini, std::shared_ptr<Layer> &layer,
- const std::string &layer_name);
-
- /**
- * @brief load backbone config from ini
- * @param[in] ini dictionary containing the config
- * @param[in] backbone_config config file containing the backbone config
- * @param[in/out] model model to be added the backbone to
- * @param[in] backbone_name name of the backbone to be loaded
- */
- int loadBackboneConfigIni(dictionary *ini, const std::string &backbone_config,
- NeuralNetwork &model,
- const std::string &backbone_name);
-
- /**
- * @brief wrapper function to load backbone config as layer from ini
- * @param[in] ini dictionary containing the config
- * @param[in] backbone_config config file containing the backbone config
- * @param[in/out] model model to be added the backbone to
- * @param[in] backbone_name name of the backbone to be loaded
- * @note External implies that this backbone is dependent on external
- * frameworks and this model will be treated as a blackbox by nntrainer.
- * Training this backbone is dependent on the API exposed by the corresponding
- * framework.
- */
- int loadBackboneConfigExternal(dictionary *ini,
- const std::string &backbone_config,
- std::shared_ptr<Layer> &layer,
- const std::string &backbone_name);
-
- /**
- * @brief Check if the file extension is the given @a ext
- * @param[in] filename full name of the file
- * @param[in] ext extension to match with
- * @retval true if @a ext, else false
- */
- static bool fileExt(const std::string &filename, const std::string &ext);
-
- /**
- * @brief Check if the file extension is ini
- * @param[in] filename full name of the file
- * @retval true if ini, else false
- */
- static bool fileIni(const std::string &filename);
-
- /**
- * @brief Check if the file extension is tflite
- * @param[in] filename full name of the file
- * @retval true if tflite, else false
- */
- static bool fileTfLite(const std::string &filename);
-
- const char *unknown = "Unknown";
-};
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __MODEL_LOADER_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file neuralnet.h
- * @date 04 December 2019
- * @brief This is Neural Network Class
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-#ifndef __NEURALNET_H__
-#define __NEURALNET_H__
-#ifdef __cplusplus
-
-#include <memory>
-#include <vector>
-
-#include <activation_layer.h>
-#include <bn_layer.h>
-#include <conv2d_layer.h>
-#include <databuffer.h>
-#include <fc_layer.h>
-#include <flatten_layer.h>
-#include <input_layer.h>
-#include <layer_internal.h>
-#include <loss_layer.h>
-#include <ml-api-common.h>
-#include <optimizer_internal.h>
-#include <pooling2d_layer.h>
-#include <tensor.h>
-
-#include <model.h>
-#include <nntrainer-api-common.h>
-
-namespace nntrainer {
-
-/**
- * @brief Enumeration of Network Type
- */
-using NetType = ml::train::ModelType;
-
-/**
- * @brief Statistics from running or training a model
- */
-typedef struct RunStats_ {
- float accuracy; /** accuracy of the model */
- float loss; /** loss of the model */
-
- RunStats_() : accuracy(0), loss(0) {}
-} RunStats;
-
-/**
- * @class NeuralNetwork Class
- * @brief NeuralNetwork Class which has Network Configuration & Layers
- */
-class NeuralNetwork : public ml::train::Model {
- friend class ModelLoader; /** access private members of ModelLoader */
-
-public:
- using NodeType = std::shared_ptr<Layer>; /** Type of a Node */
- using GraphType = std::vector<NodeType>; /** actual graph type */
- using FlatGraphType =
- std::vector<NodeType>; /** topological sorted, iterable 1-D list of nodes */
-
- /**
- * @brief Constructor of NeuralNetwork Class
- */
- NeuralNetwork() :
- batch_size(1),
- epochs(1),
- epoch_idx(0),
- iter(0),
- loss(0.0f),
- loss_type(LossType::LOSS_UNKNOWN),
- weight_initializer(WeightInitializer::WEIGHT_UNKNOWN),
- net_type(NetType::UNKNOWN),
- data_buffer(nullptr),
- continue_train(false),
- initialized(false),
- def_name_count(0),
- loadedFromConfig(false) {}
-
- /**
- * @brief Destructor of NeuralNetwork Class
- */
- ~NeuralNetwork();
-
- /**
- * @brief Get Loss from the previous ran batch of data
- * @retval loss value
- */
- float getLoss();
-
- /**
- * @brief Get Loss from the previous epoch of training data
- * @retval loss value
- */
- float getTrainingLoss() { return training.loss; }
-
- /**
- * @brief Get Loss from the previous epoch of validation data
- * @retval loss value
- */
- float getValidationLoss() { return validation.loss; }
-
- /**
- * @brief Get Learning rate
- * @retval Learning rate
- */
- float getLearningRate() { return opt->getLearningRate(); };
-
- /**
- * @brief Create and load the Network with ini configuration file.
- * @param[in] config config file path
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int loadFromConfig(std::string config);
-
- /**
- * @brief set Property of Network
- * @param[in] values values of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(std::vector<std::string> values);
-
- /**
- * @brief Initialize Network. This should be called after set all
- * hyperparameters.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int init();
-
- /**
- * @brief Forward Propagation of the neural network
- * @param[in] input List of Input Tensors taken by the neural network
- * @retval List of Output Tensors
- */
- sharedConstTensors forwarding(sharedConstTensors input);
-
- /**
- * @brief Forward Propagation of the neural network
- * @param[in] input List of Input Tensors taken by the neural network
- * @param[in] label List of Label Tensors for the model
- * @retval List of Output Tensors
- */
- sharedConstTensors forwarding(sharedConstTensors input,
- sharedConstTensors label);
-
- /**
- * @brief Backward Propagation of the neural network
- * @param[in] input List of Input Tensors taken by the neural network
- * @param[in] label List of Label Tensors for the model
- * @param[in] iteration Iteration Number for the optimizer
- */
- void backwarding(sharedConstTensors input, sharedConstTensors label,
- int iteration);
-
- /**
- * @brief save model and training parameters into file
- */
- void saveModel();
-
- /**
- * @brief read model and training parameters from file
- */
- void readModel();
-
- /**
- * @brief get Epochs
- * @retval epochs
- */
- unsigned int getEpochs() { return epochs; };
-
- /**
- * @brief Copy Neural Network
- * @param[in] from NeuralNetwork Object to copy
- * @retval NeuralNewtork Object copyed
- */
- NeuralNetwork ©(NeuralNetwork &from);
-
- /**
- * @brief Run NeuralNetwork train
- * @param[in] values hyper parameters
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int train(std::vector<std::string> values = {});
-
- /**
- * @brief Run NeuralNetwork inference
- * @param[in] X input tensor
- * @retval shared_ptr<const Tensor>
- */
- sharedConstTensors inference(sharedConstTensors X);
-
- /**
- * @brief Run NeuralNetwork train with callback function by user
- * @param[in] dataset set the dataset
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setDataset(std::shared_ptr<ml::train::Dataset> dataset) {
- return setDataBuffer(std::static_pointer_cast<DataBuffer>(dataset));
- }
-
- /**
- * @brief Run NeuralNetwork train with callback function by user
- * @param[in] databuffer set the databuffer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setDataBuffer(std::shared_ptr<DataBuffer> data_buffer);
-
- /**
- * @brief add layer into neural network model
- * @param[in] layer layer to add
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int addLayer(std::shared_ptr<ml::train::Layer> layer) {
- return addLayer(std::static_pointer_cast<Layer>(layer));
- }
-
- /**
- * @brief add layer into neural network model
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int addLayer(NodeType layer);
-
- /**
- * @brief join passed graph into the existing graph model
- * @param[in] graph graph to be added/to extend
- * @param[in] prefix prefix added to names of layers from this graph
- * @note It is assumed that this model is valid by itself
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int extendGraph(GraphType graph, std::string prefix = "");
-
- /**
- * @brief set optimizer for the neural network model
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setOptimizer(std::shared_ptr<ml::train::Optimizer> optimizer);
-
- /*
- * @brief get layer by name from neural network model
- * @param[in] name name of the layer to get
- * @param[out] layer shared_ptr to hold the layer to get
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int getLayer(const char *name, std::shared_ptr<ml::train::Layer> *layer);
-
- /*
- * @brief get layer by name from neural network model
- * @param[in] name name of the layer to get
- * @param[out] layer shared_ptr to hold the layer to get
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int getLayer(const char *name, NodeType *layer);
-
- /*
- * @brief get input dimension of neural network
- * @retval std::vector<TensorDim> input dimension
- */
- std::vector<TensorDim> getInputDimension() {
- return layers[0]->getInputDimension();
- }
-
- /*
- * @brief get output dimension of neural network
- * @retval std::vector<TensorDim> output dimension
- */
- std::vector<TensorDim> getOutputDimension() {
- return layers.back()->getOutputDimension();
- }
-
- /**
- * @brief get FlatGraph of current graph
- * @note flat graph contains pointer to the actual nodes, which is not deeply
- * copied.
- * @retval flatGraph of the current graph
- */
- FlatGraphType getFlatGraph() { return layers; }
-
- /**
- * @brief get current graph from the model
- * @note graph contains pointer to the actual nodes, which is not deeply
- * copied.
- * @retval current graph
- */
- GraphType getGraph() { return layers; }
-
- /**
- * @brief Set loss type for the neural network.
- * @param[in] loss Type of the loss.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setLoss(LossType loss);
-
- /**
- * @brief Print Option when printing model info. The function delegates to the
- * `print`
- * @param out std::ostream to print
- * @param preset preset from `ml_train_summary_type_e`
- */
- void printPreset(std::ostream &out, unsigned int preset);
-
-private:
- /**
- * @brief Print Options when printing layer info
- */
- typedef enum {
- // clang-format off
- PRINT_INST_INFO = (1 << 0), /**< Option to print type & instance address info */
- PRINT_GRAPH_INFO = (1 << 1), /**< Option to print graph topology info */
- PRINT_PROP = (1 << 2), /**< Option to print properties */
- PRINT_OPTIMIZER = (1 << 3), /**< Option to print optimizer */
- PRINT_METRIC = (1 << 4), /**< Option to print if current network is set to training */
- // clang-format on
- } PrintOption;
-
- unsigned int batch_size; /**< batch size */
-
- unsigned int epochs; /**< Maximum Epochs */
-
- unsigned int epoch_idx; /**< Number of epoch_idx */
-
- unsigned int iter; /**< iterations trained */
-
- float loss; /**< loss */
-
- LossType loss_type; /**< Loss Function type */
-
- WeightInitializer weight_initializer; /**< Weight Initialization type */
-
- std::string save_path; /**< Model path to save / read */
-
- std::shared_ptr<Optimizer> opt; /**< Optimizer; this gets copied into each
- layer, do not use this directly */
-
- NetType net_type; /**< Network Type */
-
- GraphType layers; /**< vector for store layer pointers */
-
- std::shared_ptr<DataBuffer> data_buffer; /**< Data Buffer to get Input */
-
- bool continue_train; /**< Continue train from the previous state of optimizer
- and iterations */
-
- bool initialized; /**< Network is initialized */
-
- std::set<std::string>
- layer_names; /**< Set containing all the names of layers in the model */
-
- int def_name_count; /**< Count assigned to layer names declared by default */
-
- bool loadedFromConfig; /**< Check if config is loaded to prevent load twice */
-
- RunStats validation; /** validation statistics of the model */
- RunStats training; /** training statistics of the model */
- RunStats testing; /** testing statistics of the model */
-
- /**
- * @brief print function for neuralnet
- * @param[in] out outstream
- * @param[in] flags bit combination of Neuralnet::PrintOption
- * @param[in] Layer::PrintPreset print preset when to print layer properties
- */
- void print(
- std::ostream &out, unsigned int flags = 0,
- Layer::PrintPreset layerPrintPreset = Layer::PrintPreset::PRINT_SUMMARY);
-
- /**
- * @brief Sets up and initialize the loss layer
- */
- int initLossLayer();
-
- /**
- * @brief Set Loss
- * @param[in] l loss value
- */
- void setLoss(float l);
-
- /**
- * @brief Run NeuralNetwork train
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int train_run();
-
- /**
- * @brief check neural network is ready to init.
- * @retval #ML_ERROR_NONE neuralnetwork is ready to init
- * @retval #ML_ERROR_INVALID_PARAMETER not ready to init.
- */
- int isInitializable();
-
- /**
- * @brief Realize act type to layer and insert it to layers
- * @param[in] ActivationType act Activation Type
- * @param[in] int Position position to insert activation layer.
- * @note layer is inserted at position
- */
- int realizeActivationType(const ActivationType act,
- const unsigned int position);
-
- /**
- * @copydoc int realizeActivationType(ActivationType act, unsigned int
- * &position);
- * @note layer is inserted at the back of layers
- */
- int realizeActivationType(const ActivationType act);
-
- /**
- * @brief Realize flatten type to layer and insert it to layers
- * @param[in] int Position position to insert the layer.
- * @note layer is inserted at position
- */
- int realizeFlattenType(const unsigned int position);
-
- /**
- * @copydoc int realizeActivationType(ActivationType act, unsigned int
- * &position);
- * @note layer is inserted at the back of layers
- */
- int realizeFlattenType();
-
- /**
- * @brief Ensure that layer has a name
- */
- void ensureName(NodeType layer, const std::string &prefix = "",
- bool force_rename = false);
-
- /**
- * @brief Swap function for the class
- */
- friend void swap(NeuralNetwork &lhs, NeuralNetwork &rhs) {
- using std::swap;
-
- swap(lhs.batch_size, rhs.batch_size);
- swap(lhs.epochs, rhs.epochs);
- swap(lhs.epoch_idx, rhs.epoch_idx);
- swap(lhs.iter, rhs.iter);
- swap(lhs.loss, rhs.loss);
- swap(lhs.loss_type, rhs.loss_type);
- swap(lhs.weight_initializer, rhs.weight_initializer);
- swap(lhs.save_path, rhs.save_path);
- swap(lhs.opt, rhs.opt);
- swap(lhs.net_type, rhs.net_type);
- swap(lhs.layers, rhs.layers);
- swap(lhs.data_buffer, rhs.data_buffer);
- swap(lhs.continue_train, rhs.continue_train);
- swap(lhs.initialized, rhs.initialized);
- swap(lhs.layer_names, rhs.layer_names);
- swap(lhs.def_name_count, rhs.def_name_count);
- swap(lhs.loadedFromConfig, rhs.loadedFromConfig);
- }
-
- /**
- * @brief set Property/Configuration of Network for training after the
- * network has been initialized
- * @param[in] values values of property
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setTrainConfig(std::vector<std::string> values);
-
- /**
- * @brief Update batch size of the model as well as its layers/dataset
- */
- void setBatchSize(unsigned int batch_size);
-
- /**
- * @brief print metrics function for neuralnet
- * @param[in] out outstream
- * @param[in] flags verbosity from ml_train_summary_type_e
- */
- void printMetrics(std::ostream &out, unsigned int flags = 0);
-};
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __NEURALNET_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file nnstreamer_layer.h
- * @date 26 October 2020
- * @brief This is class to encapsulate nnstreamer as a layer of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug no known bugs except for NYI items
- *
- */
-
-#ifndef __NNSTREAMER_LAYER_H__
-#define __NNSTREAMER_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <nnstreamer-single.h>
-#include <nnstreamer.h>
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @class NNStreamerLayer
- * @brief nnstreamer layer
- */
-class NNStreamerLayer : public Layer {
-public:
- /**
- * @brief Constructor of NNStreamer Layer
- */
- NNStreamerLayer(std::string model = "") :
- Layer(LayerType::LAYER_BACKBONE_NNSTREAMER),
- modelfile(model),
- single(nullptr),
- in_res(nullptr),
- out_res(nullptr),
- in_data_cont(nullptr),
- out_data_cont(nullptr) {
- trainable = false;
- }
-
- /**
- * @brief Destructor of NNStreamer Layer
- */
- ~NNStreamerLayer();
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @copydoc Layer::copy(std::shared_ptr<layer> l)
- */
- void copy(std::shared_ptr<Layer> l);
-
- /**
- * @copydoc Layer::initialize()
- */
- int initialize();
-
- /**
- * @copydoc Layer::setTrainable(bool train)
- */
- void setTrainable(bool train);
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "BackboneNNStreamer"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- std::string modelfile;
- ml_single_h single;
- ml_tensors_info_h in_res, out_res;
- ml_tensors_data_h in_data_cont, out_data_cont;
- void *in_data, *out_data;
-
- /**
- * @brief finalize the layer with the given status
- * @param[in] status status to return
- * @retval return status received as argument
- */
- int finalizeError(int status);
-
- /**
- * @brief convert nnstreamer's tensor_info to nntrainer's tensor_dim
- * @param[in] out_res nnstreamer's tensor_info
- * @param[out] dim nntrainer's tensor_dim
- * @retval 0 on success, -errno on failure
- */
- static int nnst_info_to_tensor_dim(ml_tensors_info_h &out_res,
- TensorDim &dim);
-};
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __NNSTREAMER_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file nntrainer_error.h
- * @date 03 April 2020
- * @brief NNTrainer Error Codes
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- */
-
-#ifndef __NNTRAINER_ERROR_H__
-#define __NNTRAINER_ERROR_H__
-
-#include <ml-api-common.h>
-#if defined(__TIZEN__)
-#include <tizen_error.h>
-#define ML_ERROR_BAD_ADDRESS TIZEN_ERROR_BAD_ADDRESS
-#define ML_ERROR_RESULT_OUT_OF_RANGE TIZEN_ERROR_RESULT_OUT_OF_RANGE
-#else
-#include <cerrno>
-#define ML_ERROR_BAD_ADDRESS (-EFAULT)
-#define ML_ERROR_RESULT_OUT_OF_RANGE (-ERANGE)
-#endif
-
-#include <stdexcept>
-namespace nntrainer {
-
-/// @note underscore_case is used for ::exception to keep in accordance with
-/// std::exception
-namespace exception {
-
-/**
- * @brief derived class of invalid argument to represent specific functionality
- * not supported
- * @note this could be either intended or not yet implemented
- */
-struct not_supported : public std::invalid_argument {
- using invalid_argument::invalid_argument;
-};
-
-} // namespace exception
-
-} // namespace nntrainer
-
-#endif /* __NNTRAINER_ERROR_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file nntrainer_log.h
- * @date 06 April 2020
- * @brief NNTrainer Logger.
- * Log Util for NNTrainer
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- */
-
-#ifndef __NNTRAINER_LOG_H__
-#define __NNTRAINER_LOG_H__
-
-#define TAG_NAME "nntrainer"
-
-#if defined(__TIZEN__)
-#include <dlog.h>
-
-#define ml_logi(...) dlog_print(DLOG_INFO, TAG_NAME, __VA_ARGS__)
-
-#define ml_logw(...) dlog_print(DLOG_WARN, TAG_NAME, __VA_ARGS__)
-
-#define ml_loge(...) dlog_print(DLOG_ERROR, TAG_NAME, __VA_ARGS__)
-
-#define ml_logd(...) dlog_print(DLOG_DEBUG, TAG_NAME, __VA_ARGS__)
-
-#elif defined(__ANDROID__)
-#include <android/log.h>
-
-#define ml_logi(...) \
- __android_log_print(ANDROID_LOG_INFO, TAG_NAME, __VA_ARGS__)
-
-#define ml_logw(...) \
- __android_log_print(ANDROID_LOG_WARN, TAG_NAME, __VA_ARGS__)
-
-#define ml_loge(...) \
- __android_log_print(ANDROID_LOG_ERROR, TAG_NAME, __VA_ARGS__)
-
-#define ml_logd(...) \
- __android_log_print(ANDROID_LOG_DEBUG, TAG_NAME, __VA_ARGS__)
-
-#else /* Linux distro */
-#include <nntrainer_logger.h>
-
-#if !defined(ml_logi)
-#define ml_logi(format, args...) \
- __nntrainer_log_print(NNTRAINER_LOG_INFO, "(%s:%s:%d) " format, __FILE__, \
- __func__, __LINE__, ##args)
-#endif
-
-#if !defined(ml_logw)
-#define ml_logw(format, args...) \
- __nntrainer_log_print(NNTRAINER_LOG_WARN, "(%s:%s:%d) " format, __FILE__, \
- __func__, __LINE__, ##args)
-#endif
-
-#if !defined(ml_loge)
-#define ml_loge(format, args...) \
- __nntrainer_log_print(NNTRAINER_LOG_ERROR, "(%s:%s:%d) " format, __FILE__, \
- __func__, __LINE__, ##args)
-#endif
-
-#if !defined(ml_logd)
-#define ml_logd(format, args...) \
- __nntrainer_log_print(NNTRAINER_LOG_DEBUG, "(%s:%s:%d) " format, __FILE__, \
- __func__, __LINE__, ##args)
-#endif
-
-#endif
-
-#endif /* __NNTRAINER_LOG_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file nntrainer_logger.h
- * @date 02 April 2020
- * @brief NNTrainer Logger
- * This allows to logging nntrainer logs.
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- */
-#ifndef __NNTRAINER_LOGGER_H___
-#define __NNTRAINER_LOGGER_H___
-#ifdef __cplusplus
-
-#include <fstream>
-#include <mutex>
-#include <string>
-
-/**
- * @brief Log Level of NNtrainer
- * 0. informations
- * 1. warnings
- * 2. errors
- * 3. debugging informations
- */
-typedef enum {
- NNTRAINER_LOG_DEBUG = 0,
- NNTRAINER_LOG_INFO,
- NNTRAINER_LOG_WARN,
- NNTRAINER_LOG_ERROR
-} nntrainer_loglevel;
-
-namespace nntrainer {
-
-/**
- * @class NNTrainer Logger Class
- * @brief Class for Logging. This is alternatives when there is no logging
- * system. For the tizen, we are going to use dlog and it is android_log for
- * android.
- */
-class Logger {
-public:
- /**
- * @brief Logging Instance Function. Get a lock and create Logger if it
- * is null;
- */
- static Logger &instance();
-
- /**
- * @brief Logging member function for logging messages.
- */
- void log(const std::string &message,
- const nntrainer_loglevel loglevel = NNTRAINER_LOG_INFO);
-
-protected:
- /**
- * @brief Logging instance
- */
- static Logger *ainstance;
- /**
- * @brief Log file name
- */
- static const char *const logfile_name;
-
- /**
- * @brief output stream
- */
- std::ofstream outputstream;
-
- /**
- * @class Class to make sure the single logger
- * @brief sure the single logger
- */
- friend class Cleanup;
- class Cleanup {
- public:
- ~Cleanup();
- };
-
-private:
- /**
- * @brief Constructor
- */
- Logger();
- /**
- * @brief Destructor
- */
- virtual ~Logger();
- Logger(const Logger &);
- Logger &operator=(const Logger &);
- static std::mutex smutex;
-};
-} /* namespace nntrainer */
-
-extern "C" {
-
-#endif /* __cplusplus */
-
-/**
- * @brief Interface function for C
- */
-void __nntrainer_log_print(nntrainer_loglevel loglevel,
- const std::string format, ...);
-
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-
-#endif /* __NNTRAINER_LOGGER_H___ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file optimizer_factory.h
- * @date 7 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the optimizer factory.
- */
-
-#ifndef __OPTIMIZER_FACTORY_H__
-#define __OPTIMIZER_FACTORY_H__
-#ifdef __cplusplus
-
-#include <optimizer_internal.h>
-
-namespace nntrainer {
-
-/**
- * @brief Factory creator with copy constructor
- */
-std::unique_ptr<Optimizer> createOptimizer(OptType type, const Optimizer &opt);
-
-/**
- * @brief Factory creator with constructor
- */
-std::unique_ptr<Optimizer> createOptimizer(OptType type);
-
-} // namespace nntrainer
-
-#endif // __cplusplus
-#endif // __OPTIMIZER_FACTORY_H__
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file optimizer_internal.h
- * @date 08 April 2020
- * @brief This is Optimizer classes of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-#ifndef __OPTIMIZER_H__
-#define __OPTIMIZER_H__
-#ifdef __cplusplus
-
-#include <memory>
-#include <optimizer.h>
-#include <tensor.h>
-#include <weight.h>
-
-namespace nntrainer {
-
-/**
- * @brief Enumeration of Optimizer
- * 0. SGD
- * 1. ADAM
- * 2. Unknown
- */
-using OptType = ml::train::OptimizerType;
-
-/**
- * @class Optimizer Base class for optimizers
- * @brief Base class for all optimizers
- */
-class Optimizer : public ml::train::Optimizer {
-
- /** Allow layer to initialize optimizer with itself */
- friend class Layer;
-
-public:
- /**
- * @brief Default Constructor of Optimizer Class
- */
- Optimizer(const OptType t, float lr, float decay_rate = 1.0f,
- unsigned int decay_steps = 0, float continue_train = false) :
- type(t),
- learning_rate(lr),
- decay_rate(decay_rate),
- decay_steps(decay_steps),
- continue_train(continue_train) {
- checkValidation();
- }
-
- /**
- * @brief copy constructor
- * @param[in] rhs Optimizer to be copied
- */
- Optimizer(const Optimizer &rhs) = default;
-
- /**
- * @brief copy assignment operator
- * @param[in] rhs Optimizer to be copied
- */
- Optimizer &operator=(const Optimizer &rhs) = default;
-
- /**
- * @brief Move constructor of Conv 2D Layer.
- * @param[in] Conv2dLayer &&
- */
- Optimizer(Optimizer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs Optimizer to be moved.
- */
- Optimizer &operator=(Optimizer &&rhs) = default;
-
- /**
- * @brief get Optimizer Type
- * @retval Optimizer type
- */
- OptType getType() { return type; };
-
- /**
- * @brief get Learning Rate
- * @retval Learning rate
- */
- float getLearningRate() { return learning_rate; };
-
- /**
- * @brief get Decay Rate for learning rate decay
- * @retval decay rate
- */
- float getDecayRate() { return decay_rate; };
-
- /**
- * @brief get Decay Steps for learning rate decay
- * @retval decay steps
- */
- float getDecaySteps() { return decay_steps; };
-
- /**
- * @brief set Optimizer Parameters
- * @param[in] values Optimizer Parameter list
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setProperty(std::vector<std::string> values);
-
- /**
- * @brief apply gradient to weight_list
- * @param[in] params Weight list
- * @param[in] num_weights size of the array
- * @param[in] iteration nth epoch number
- */
- void apply_gradients(std::shared_ptr<Weight> params, unsigned int num_weights,
- int iteration);
-
- /**
- * @brief Read Training optimizer paramters from file
- * @param[in] file input stream file
- */
- virtual void read(std::ifstream &file);
-
- /**
- * @brief Save Training optimizer paramters from file
- * @param[in] file output stream file
- */
- virtual void save(std::ofstream &file);
-
- /**
- * @brief setProperty by PropertyType
- * @note By passing empty string, this can validate if @a type is valid
- * @param[in] type property type to be passed
- * @param[in] value value to be passed, if empty string is passed, do nothing
- * but throws error when @a type is invalid
- * @exception exception::not_supported when property type is not valid for
- * the particular layer
- * @exception std::invalid_argument invalid argument
- */
- virtual void setProperty(const PropertyType type,
- const std::string &value = "");
-
- /**
- * @brief get the base name for the optimizer
- * @retval base name of the optimizer
- */
- virtual std::string getBaseName() = 0;
-
- /**
- * @brief validate the optimizer
- */
- virtual void checkValidation();
-
-protected:
- /**
- * @brief Optimizer Type
- */
- OptType type;
-
- /**
- * @brief get Learning Rate for the given iteration
- * @param[in] iteration Iteration for the learning rate
- * @retval Learning rate
- */
- virtual double getLearningRate(int iteration);
-
- float learning_rate; /** learning rate */
- float decay_rate; /** decay rate for learning rate */
- unsigned int decay_steps; /** decay steps for learning rate */
- bool continue_train; /** Continue training with previous tensors for adam */
-
-private:
- /**
- * @brief initialize optimizer. Initialize Weight if it is adam
- * @param[in] params Weight list
- * @param[in] num_weights size of the array
- * @param[in] setTensor true if the layer need weight update.
- * Input Layer and Batch Normalization layer won't need it.
- * Therefore, it sets false.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int initialize(std::shared_ptr<Weight> params,
- unsigned int num_weights, bool setTensor);
-
- /**
- * @brief apply gradient to the given weight
- * @param[in] weight Weight and gradient set to be updated
- * @param[in] tensor_idx Idx of this tensor in the tensors list
- * @param[in] num_weights size of the array
- * @param[in] iteration nth epoch number
- * @note weight which is called upon can be assumed to be trainable
- */
- virtual void apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- int iteration) = 0;
-};
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __OPTIMIZER_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file parse_util.h
- * @date 07 May 2020
- * @brief This is collection of parse functions.
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __PARSE_UTIL_H__
-#define __PARSE_UTIL_H__
-#ifdef __cplusplus
-
-#include <iostream>
-#include <regex>
-#include <string>
-#include <vector>
-
-namespace nntrainer {
-
-#define NN_RETURN_STATUS() \
- do { \
- if (status != ML_ERROR_NONE) { \
- return status; \
- } \
- } while (0)
-
-/**
- * @brief Enumeration for input configuration file parsing
- * 0. OPT ( Optimizer Token )
- * 1. LOSS ( Loss Function Token )
- * 2. MODEL ( Model Token )
- * 3. ACTI ( Activation Token )
- * 4. LAYER ( Layer Token )
- * 5. WEIGHT_INIT ( Weight Initializer Token )
- * 7. WEIGHT_REGULARIZER ( Weight Decay Token )
- * 8. PADDING ( Padding Token )
- * 9. POOLING ( Pooling Token )
- * 9. UNKNOWN
- */
-typedef enum {
- TOKEN_OPT,
- TOKEN_LOSS,
- TOKEN_MODEL,
- TOKEN_ACTI,
- TOKEN_LAYER,
- TOKEN_WEIGHT_INIT,
- TOKEN_WEIGHT_REGULARIZER,
- TOKEN_PADDING,
- TOKEN_POOLING,
- TOKEN_UNKNOWN
-} InputType;
-
-inline void throw_status(int status) {
- switch (status) {
- case ML_ERROR_NONE:
- break;
- case ML_ERROR_INVALID_PARAMETER:
- throw std::invalid_argument("invalid argument from c style throw");
- case ML_ERROR_OUT_OF_MEMORY:
- throw std::bad_alloc();
- case ML_ERROR_TIMED_OUT:
- throw std::runtime_error("Timed out from c style throw");
- case ML_ERROR_PERMISSION_DENIED:
- throw std::runtime_error("permission denied from c style throw");
- case ML_ERROR_UNKNOWN:
- default:
- throw std::runtime_error("unknown error from c style throw");
- }
-}
-/**
- * @brief Parsing Layer Property
- * @param[in] property string to be parsed
- * @retval int enumerated type
- */
-unsigned int parseLayerProperty(std::string property);
-
-/**
- * @brief Unparse Layer property to string
- * @param[in] type property type
- * @retval string representation of the type
- */
-std::string propToStr(const unsigned int type);
-
-/**
- * @brief Parsing Configuration Token
- * @param[in] ll string to be parsed
- * @param[in] t Token type
- * @retval int enumerated type
- */
-unsigned int parseType(std::string ll, InputType t);
-
-/**
- * @brief Parsing Optimizer Property
- * @param[in] property string to be parsed
- * @retval int enumerated type
- */
-unsigned int parseOptProperty(std::string property);
-
-/**
- * @brief Parsing Network Property
- * @param[in] property string to be parsed
- * @retval int enumerated type
- */
-unsigned int parseNetProperty(std::string property);
-
-/**
- * @brief Parsing Data Buffer Property
- * @param[in] property string to be parsed
- * @retval int enumerated type
- */
-unsigned int parseDataProperty(std::string property);
-
-/**
- * @brief check str to be unsigned int and assign to variable to type T
- * @param[out] val assign variable
- * @param[in] str input string
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
-int setUint(unsigned int &val, const std::string &str);
-
-/**
- * @brief check str to be float and assign
- * @param[out] val assign variable
- * @param[in] str input string
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
-int setFloat(float &val, std::string str);
-
-/**
- * @brief check str to be double and assign
- * @param[out] val assign variable
- * @param[in] str input string
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
-int setDouble(double &val, std::string str);
-
-/**
- * @brief check str to be bool and assign
- * @param[out] val assign variable
- * @param[in] str input string
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
-int setBoolean(bool &val, std::string str);
-
-/**
- * @brief parse string and return key & value
- * @param[in] input_str input string to split with '='
- * @param[out] key key
- * @param[out] value value
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
-int getKeyValue(std::string input_str, std::string &key, std::string &value);
-
-/**
- * @brief join vector of int to string with delimiter ","
- * @param[in] values vector of int
- * @param[in] delimiter delimiter for the string
- * @retval output string
- */
-const char *getValues(std::vector<int> values, const char *delimiter = ",");
-
-int getValues(int n_str, std::string str, int *value);
-
-/**
- * @brief split string into vector with delimiter regex
- * @param[in] str string
- * @param[in] reg regular expression to use as delimiter
- * @retval output string vector
- */
-std::vector<std::string> split(const std::string &s, std::regex ®);
-
-/**
- * @brief print instance info. as <Type at (address)>
- * @param[in] std::ostream &out, T&& t
- * @param[in] t pointer to the instance
- */
-template <typename T,
- typename std::enable_if_t<std::is_pointer<T>::value, T> * = nullptr>
-void printInstance(std::ostream &out, const T &t) {
- out << '<' << typeid(*t).name() << " at " << t << '>' << std::endl;
-}
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __PARSE_UTIL_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file pooling2d_layer.h
- * @date 12 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is 2 Dimensional Pooling Layer Class for Neural Network
- *
- */
-
-#ifndef __POOLING2D_LAYER_H__
-#define __POOLING2D_LAYER_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-#include <vector>
-
-#define POOLING2D_DIM 2
-
-namespace nntrainer {
-
-/**
- * @class Pooling 2D Layer
- * @brief Pooling 2D Layer
- */
-class Pooling2DLayer : public Layer {
-public:
- enum class PoolingType {
- max = 0,
- average = 1,
- global_max = 2,
- global_average = 3,
- unknown = 4,
- };
-
- /**
- * @brief Constructor of Pooling 2D Layer
- */
- template <typename... Args>
- Pooling2DLayer(
- PoolingType pooling_type_ = PoolingType::average,
- const std::array<unsigned int, POOLING2D_DIM> &pool_size_ = {0, 0},
- const std::array<unsigned int, POOLING2D_DIM> &stride_ = {1, 1},
- const std::array<unsigned int, POOLING2D_DIM> &padding_ = {0, 0},
- Args... args) :
- Layer(LayerType::LAYER_POOLING2D, args...),
- pool_size(pool_size_),
- stride(stride_),
- padding(padding_),
- pooling_type(pooling_type_) {}
-
- /**
- * @brief Destructor of Pooling 2D Layer
- */
- ~Pooling2DLayer() {}
-
- /**
- * @brief Move constructor of Pooling 2D Layer.
- * @param[in] Pooling2D &&
- */
- Pooling2DLayer(Pooling2DLayer &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs Pooling2DLayer to be moved.
- */
- Pooling2DLayer &operator=(Pooling2DLayer &&rhs) = default;
-
- /**
- * @brief initialize layer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int initialize();
-
- /**
- * @brief Read Weight & Bias Data from file
- * @param[in] file input stream file
- */
- void read(std::ifstream &file){};
-
- /**
- * @brief Save Weight & Bias Data to file
- * @param[in] file output stream file
- */
- void save(std::ofstream &file){};
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @copydoc Layer::setBatch(unsigned int batch)
- */
- void setBatch(unsigned int batch);
-
- /**
- * @brief copy layer
- * @param[in] l layer to copy
- */
- void copy(std::shared_ptr<Layer> l);
-
- /* TO DO : support keras type of padding */
- enum class PaddingType {
- full = 0,
- same = 1,
- valid = 2,
- unknown = 3,
- };
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "Pooling2D"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- std::array<unsigned int, POOLING2D_DIM> pool_size;
- std::array<unsigned int, POOLING2D_DIM> stride;
- std::array<unsigned int, POOLING2D_DIM> padding;
- std::vector<unsigned int> max_idx;
- std::vector<std::vector<unsigned int>> max_idx_global;
- PoolingType pooling_type;
-
- /**
- * @brief calculation convolution
- * @param[in] batch batch index
- * @param[in] in input tensor
- * @retval Tensor outoput tensor
- */
- Tensor pooling2d(unsigned int batch, Tensor &in);
-
- /**
- * @brief set Pooling Type
- * @param[in] t pooling type
- */
- void setPoolingType(PoolingType t) { pooling_type = t; };
-
- /**
- * @brief set Parameter Size
- * @param[in] * size : size arrary
- * @param[in] type : Property type
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setSize(int *size, PropertyType type);
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __POOLING_LAYER_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file sgd.h
- * @date 6 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the SGD optimizer.
- */
-#ifndef __SGD_H__
-#define __SGD_H__
-#ifdef __cplusplus
-
-#include <optimizer_internal.h>
-
-namespace nntrainer {
-
-/**
- * @class SGD optimizer class
- * @brief Stochastic Gradient Descent optimizer class
- */
-class SGD : public Optimizer {
-public:
- /**
- * @brief Constructor of Optimizer Class
- */
- template <typename... Args>
- SGD(float lr = 0.0001f, Args... args) :
- Optimizer(OptType::SGD, lr, args...) {}
-
- /**
- * @copydoc apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- * int iteration)
- */
- void apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- int iteration);
-
- /**
- * @brief get the base name for the optimizer
- * @retval base name of the optimizer
- */
- std::string getBaseName() { return "SGD"; };
-};
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __SGD_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file tensor.h
- * @date 04 December 2019
- * @brief This is Tensor class for calculation
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __TENSOR_H__
-#define __TENSOR_H__
-#ifdef __cplusplus
-
-#include <array>
-#include <functional>
-#include <memory>
-#include <vector>
-
-#include <tensor_dim.h>
-
-#define MAKE_SHARED_TENSOR(...) std::make_shared<nntrainer::Tensor>(__VA_ARGS__)
-
-namespace nntrainer {
-
-class LazyTensor;
-
-/**
- * @class Tensor Class for Calculation
- * @brief Tensor Class for Calculation
- */
-class Tensor {
-public:
- /**
- * @brief Basic Constructor of Tensor
- */
- Tensor() :
- dim(TensorDim()),
- strides{dim.computeStrides()},
- is_contiguous(true),
- data(nullptr) {}
-
- /**
- * @brief Constructor of Tensor with dimension/buf
- */
- Tensor(const TensorDim &d, const float *buf = nullptr);
-
- /**
- * @brief Constructor of Tensor
- * @param[in] batch Batch of Tensor
- * @param[in] channel Channel of Tensor
- * @param[in] height Height of Tensor
- * @param[in] width Width of Tensor
- */
- Tensor(int batch, int channel, int height, int width) :
- Tensor(TensorDim(batch, channel, height, width)){};
-
- /**
- * @brief Constructor of Tensor
- * @param[in] channel Channel of Tensor
- * @param[in] height Height of Tensor
- * @param[in] width Width of Tensor
- */
- Tensor(int channel, int height, int width) :
- Tensor(1, channel, height, width){};
-
- /**
- * @brief Constructor of Tensor with batch size one and channel size one
- * @param[in] height Height of Tensor
- * @param[in] width Width of Tensor
- */
- Tensor(int height, int width) : Tensor(1, 1, height, width){};
-
- /**
- * @brief Constructor of Tensor with just width
- * @param[in] width Width of Tensor
- */
- Tensor(int width) : Tensor(1, 1, 1, width){};
-
- /**
- * @brief Constructor of Tensor
- * @param[in] d data for the Tensor
- */
- Tensor(std::vector<std::vector<std::vector<std::vector<float>>>> const &d);
-
- /**
- * @brief Constructor of Tensor
- * @note This constructor copies vector again. needs refactoring
- * @param[in] d data for the Tensor
- */
- Tensor(std::vector<std::vector<std::vector<float>>> const &d) :
- Tensor(std::vector<std::decay<decltype(d)>::type>{d}){};
-
- /**
- * @brief Constructor of Tensor
- * @note This constructor copies vector again. needs refactoring
- * @param[in] d data for the Tensor with batch size one
- */
- Tensor(std::vector<std::vector<float>> const &d) :
- Tensor(std::vector<std::decay<decltype(d)>::type>{d}){};
-
- /**
- * @brief Copy constructor of Tensor.
- * @param[in] Tensor &
- */
- Tensor(const Tensor &rhs) = default;
-
- /**
- * @brief Move constructor of Tensor.
- * @param[in] Tensor &&
- */
- Tensor(Tensor &&rhs) noexcept = default;
-
- /**
- * @brief Copy assignment operator.
- * @param[in] rhs Tensor to be copied.
- */
- Tensor &operator=(const Tensor &rhs) = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs Tensor to be moved.
- */
- Tensor &operator=(Tensor &&rhs) noexcept = default;
-
- friend void swap(Tensor &lhs, Tensor &rhs) noexcept {
- std::swap(lhs.dim, rhs.dim);
- std::swap(lhs.data, rhs.data);
- std::swap(lhs.strides, rhs.strides);
- std::swap(lhs.is_contiguous, rhs.is_contiguous);
- }
-
- /**
- * @brief Comparison operator overload
- * @param[in] rhs Tensor to be compared with
- */
- bool operator==(const Tensor &rhs) const;
-
- /**
- * @brief Comparison operator overload
- * @param[in] rhs Tensor to be compared with
- */
- bool operator!=(const Tensor &rhs) const { return !(*this == rhs); }
-
- /**
- * @brief return value at specific location
- * @param[in] batch batch location
- * @param[in] c channel location
- * @param[in] h height location
- * @param[in] w width location
- */
- float getValue(unsigned int batch, unsigned int c, unsigned int h,
- unsigned int w) const;
-
- /**
- * @brief Multiply value element by element immediately
- * @param[in] value multiplier
- * @retval #ML_ERROR_INVALID_PARAMETER Tensor dimension is not right
- * @retval #ML_ERROR_NONE Successful
- */
- int multiply_i(float const &value);
-
- /**
- * @brief Multiply value element by element
- * @param[in] value multiplier
- * @retval Calculated Tensor
- */
- Tensor multiply(float const &value);
-
- /**
- * @brief Divide value element by element immediately
- * @param[in] value divisor
- * @retval #ML_ERROR_INVALID_PARAMETER Tensor dimension is not right
- * @retval #ML_ERROR_NONE Successful
- */
- int divide_i(float const &value);
-
- /**
- * @brief Divide value element by element
- * @param[in] value Divisor
- * @retval Calculated Tensor
- */
- Tensor divide(float const &value);
-
- /**
- * @brief Add Tensor Element by Element without mem copy
- * @param[in] m Tensor to be added
- * @param[out] alpha Values to be scaled
- * @retval #ML_ERROR_NONE Successful
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- */
- int add_i(Tensor const &m, float const alpha = 1);
-
- /**
- * @brief Add Tensor Element by Element
- * @param[in] m Tensor to be added
- * @retval Calculated Tensor
- */
- Tensor add(Tensor const &m, float const alpha = 1) const;
-
- /**
- * @brief Add Tensor Element immediately to target tensor without mem copy
- * @param[in] value value to be added
- * @retval #ML_ERROR_NONE Successful
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- */
- int add_i(float const &value);
-
- /**
- * @brief Add value Element by Element
- * @param[in] value value to be added
- * @retval Calculated Tensor
- */
- Tensor add(float const &value);
-
- /**
- * @brief memcpyless version of subtract
- * @param[in] m Tensor to be subtracted
- * @retval #ML_ERROR_NONE Successful
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- */
- int subtract_i(Tensor const &m);
-
- /**
- * @brief Substract Tensor Element by Element
- * @param[in] m Tensor to be subtracted
- * @retval Calculated Tensor
- */
- Tensor subtract(Tensor const &m) const;
-
- /**
- * @brief memcpyless version of subtract
- * @param[in] value value to subtract
- * @retval #ML_ERROR_NONE Successful
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- */
- int subtract_i(float const &value);
-
- /**
- * @brief subtract value Element by Element
- * @param[in] value value to be subtracted
- * @retval Calculated Tensor
- */
- Tensor subtract(float const &value);
-
- /**
- * @brief Multiply Tensor Elementwise
- * @param[in] m Tensor to be multiplied
- * @retval #ML_ERROR_NONE successful
- */
- int multiply_i(Tensor const &m);
-
- /**
- * @brief Multiply Tensor Element by Element ( Not the MxM )
- * @param[in] m Tensor to be multiplied
- * @retval Calculated Tensor
- */
- Tensor multiply(Tensor const &m) const;
-
- /**
- * @brief divide Tensor Elementwise
- * @param[in] m Tensor to be multiplied
- * @retval #ML_ERROR_NONE successful
- */
- int divide_i(Tensor const &m);
-
- /**
- * @brief Divide Tensor Element by Element
- * @param[in] m Divisor Tensor
- * @retval Calculated Tensor
- */
- Tensor divide(Tensor const &m) const;
-
- /**
- * @brief Tensor power Element by Element
- * @param[in] float Divisor Tensor
- * @retval Calculated Tensor
- */
- Tensor pow(float m) const;
-
- /**
- * @brief Dot Product of Tensor ( equal MxM )
- * @details This applies dot of the last dimension of this and second-last
- * dimension of passed tensor m.
- * @param[in] m Tensor
- * @param[in] trans Transpose
- * @param[in] trans_m Transpose m
- * @retval Calculated Tensor
- */
- Tensor dot(Tensor const &m, bool trans = false, bool trans_m = false) const;
-
- /**
- * @brief Transpose Tensor
- * @param[in] direction to transpose ex) 0:2:1
- * @retval Calculated Tensor
- */
- Tensor transpose(std::string direction) const;
-
- /**
- * @brief sum all the Tensor elements according to the batch
- * @retval Calculated Tensor(batch, 1, 1, 1)
- */
- Tensor sum_by_batch();
-
- /**
- * @brief sum all the Tensor elements according to the axis
- * 0 : batch direction
- * 1 : channel direction
- * 2 : height direction
- * 3 : width direction
- * @param[in] axis Axis to calculate sum along
- * @param[in] alpha Scale the sum by this value
- * @retval Calculated Tensor
- */
- Tensor sum(unsigned int axis, float alpha = 1.0) const;
-
- /**
- * @brief sum all the Tensor by multiple axes
- *
- * @param axes axes to sum along
- * @param alpha Scale the sum by this value
- * @return Tensor
- */
- Tensor sum(const std::vector<unsigned int> &axes, float alpha = 1.0) const;
-
- /**
- * @brief Averaging the Tensor elements according to the axis
- * 0 : batch direction
- * 1 : channel direction
- * 2 : height direction
- * 3 : width direction
- * @retval Calculated Tensor
- */
- Tensor average(unsigned int axis) const;
-
- /**
- * @brief average all the Tensor by multiple axes
- *
- * @param axes axes to sum along
- * @return Tensor
- */
- Tensor average(const std::vector<unsigned int> &axes) const;
-
- /**
- * @brief Averaging the Tensor elements by all axis
- * @retval Calculated Tensor
- */
- Tensor average() const;
-
- /**
- * @brief Anchor a starting point to defer following evaluation
- * @retval LazyTensor class that can be used with run();
- */
- LazyTensor chain() const;
-
- /**
- * @brief Softmax the Tensor elements
- * @retval Calculated Tensor
- */
- Tensor softmax() const;
-
- /**
- * @brief l2norm the Tensor elements
- * @retval Calculated l2norm
- */
- float l2norm() const;
-
- /**
- * @brief Normalize the Tensor elements
- * @retval Calculated Tensor
- */
- Tensor normalization() const;
-
- /**
- * @brief Standardize the Tensor elements
- * @retval Calculated Tensor
- */
- Tensor standardization() const;
-
- /**
- * @brief Fill the Tensor elements with zero
- */
- void setZero();
-
- /**
- * @brief Apply function element by element
- * @param[in] *function function pointer applied
- * @retval Tensor
- */
- Tensor apply(std::function<float(float)> f) const;
-
- /**
- * @brief Apply function to Tensor
- * @param[in] *function function pointer applied
- * @retval Tensor
- */
- Tensor apply(std::function<Tensor(Tensor)> f) const;
-
- Tensor apply_i(std::function<int(const Tensor &)> f) const;
-
- /**
- * @brief Print element
- * @param[in] out out stream
- * @retval Tensor
- */
- void print(std::ostream &out) const;
-
- /**
- * @brief Get length of current _data
- * @retval unsigned int length of the current _data
- */
- unsigned int length() const { return dim.getDataLen(); }
-
- /**
- * @brief Get size of the data
- * @retval size_t Size in bytes
- */
- size_t getSize() const { return length() * sizeof(float); }
-
- /**
- * @brief Set the element value
- * @param[in] batch batch location
- * @param[in] c channel location
- * @param[in] i height location
- * @param[in] j width location
- * @param[in] value value to be stored
- */
- void setValue(unsigned int batch, unsigned int c, unsigned int i,
- unsigned int j, float value);
-
- /**
- * @brief Fill the Tensor elements with value
- * @param[in] value value to be stored
- */
- void setValue(float value);
-
- /**
- * @brief Set the tensor with random normal distribution
- * @param[in] mean mean of the distribution
- * @param[in] std standard deviation of the distribution
- */
- void setRandNormal(float mean = 0.0f, float std = 0.05f);
-
- /**
- * @brief Set the tensor with random uniform distribution
- * @param[in] min minimum value for the distribution
- * @param[in] max maximum value for the distribution
- */
- void setRandUniform(float min = -0.05f, float max = 0.05f);
-
- /**
- * @brief Copy the Tensor
- * @param[in] from Tensor to be copied
- */
- void copy(const Tensor &from);
-
- /**
- * @brief Convient wrapper for inplace copy of @a this.
- * @retval Copied version of this
- */
- Tensor clone() const;
-
- /**
- * @brief Save the Tensor into file
- * @param[in] file output file stream
- */
- void save(std::ofstream &file);
-
- /**
- * @brief Read the Tensor from file
- * @param[in] file input file stream
- */
- void read(std::ifstream &file);
-
- /**
- * @brief return argument index which value is max by batch
- * @retval unsigned int argument index
- */
- std::vector<unsigned int> argmax() const;
-
- /**
- * @brief return a copy of the Tensor Dim
- * @retval TensorDim
- */
- TensorDim getDim() const { return TensorDim(dim); }
-
- /**
- * @brief return Tensor Dim for a given axis
- * @retval dimension
- */
- unsigned int getTensorDim(unsigned int axis);
-
- /**
- * @brief return Tensor batch size
- * @retval batch size
- */
- unsigned int batch() const { return dim.batch(); }
-
- /**
- * @brief return Tensor batch size
- * @retval batch size
- */
- unsigned int channel() const { return dim.channel(); }
-
- /**
- * @brief return Tensor height size
- * @retval height size
- */
- unsigned int height() const { return dim.height(); }
-
- /**
- * @brief return Tensor batch size
- * @retval width size
- */
- unsigned int width() const { return dim.width(); }
-
- /**
- * @brief return Data pointer of Tensor
- * @retval float pointer
- */
- float *getData() { return data.get(); }
-
- const float *getData() const { return data.get(); }
-
- /**
- * @brief i data index
- * @retval address of ith data
- */
- float *getAddress(unsigned int i);
-
- /**
- * @brief i data index
- * @retval address of ith data
- */
- const float *getAddress(unsigned int i) const;
-
- /**
- * @brief set Tensor Dim
- * @param[in] d TensorDim
- * @note Throws std::invalid_argument if size mismatch
- */
- void reshape(const TensorDim &d);
-
- /**
- * @brief return current stride of tensor.
- * @retval int[MAXDIM] strides
- */
- const std::array<unsigned int, MAXDIM> getStrides() const noexcept {
- return strides;
- }
-
- static constexpr float epsilon = 1e-5;
-
-private:
- /**
- * @brief Get linear index given the n-d index
- */
- inline unsigned int getIndex(unsigned int b, unsigned int c, unsigned int h,
- unsigned int w) const {
- return (b * strides[0] + c * strides[1] + h * strides[2] + w * strides[3]);
- }
-
- struct BroadcastInfo;
-
- /**
- * @brief Applies the given operator to the tensor with the passed argument
- * @param[in] m Tensor
- * @param[in] v_func vectorized function to apply
- * @param e broadcast info.
- * @param cur_axis current axis. pass default when calling outside.
- * @param offset offset for this. pass default when calling outside.
- * @param m_offset offset for m. pass default when calling outside.
- * @retval #ML_ERROR_NONE Successful
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- */
- int operator_i_util(
- Tensor const &m,
- std::function<void(const BroadcastInfo &e, float *, const float *)> v_func,
- const BroadcastInfo &e, int cur_axis = -1, unsigned int offset = 0,
- unsigned int m_offset = 0);
-
- /**
- * @brief Applies the given operator to the tensor with the passed argument
- *
- * @param[in] m Tensor
- * @param[in] v_func vectorized function to apply
- * @retval #ML_ERROR_NONE Successful
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- */
- int operator_i(
- Tensor const &m,
- std::function<void(const BroadcastInfo &e, float *, const float *)> v_func);
-
- /**
- * @brief compute Loop info for broadcasting and vectorization
- *
- * @param m target tensor to be calculated against.
- * @return BroadcastInfo Loopinfo needed to run external loop
- */
- BroadcastInfo computeBroadcastInfo(const Tensor &m);
-
- /**< handle the data as a std::shared_ptr<float> type */
- TensorDim dim;
- std::array<unsigned int, MAXDIM> strides;
- bool is_contiguous;
- std::shared_ptr<float> data;
-
- template <typename T> void setDist(T dist);
-};
-
-/**
- * @brief Overriding output stream
- */
-std::ostream &operator<<(std::ostream &out, Tensor const &m);
-
-typedef std::shared_ptr<Tensor> sharedTensor;
-
-typedef std::shared_ptr<const Tensor> sharedConstTensor;
-
-typedef std::vector<sharedConstTensor> sharedConstTensors;
-
-typedef std::vector<sharedTensor> sharedTensors;
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __TENSOR_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- */
-/**
- * @file tensor_dim.h
- * @date 22 May 2020
- * @brief This is Tensor Dimension Class
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __TENSOR_DIM_H__
-#define __TENSOR_DIM_H__
-#ifdef __cplusplus
-
-#include <array>
-#include <iostream>
-
-namespace nntrainer {
-
-constexpr const size_t MAXDIM = 4;
-
-class TensorDim {
-public:
- TensorDim() {
- for (size_t i = 0; i < MAXDIM; ++i) {
- dim[i] = 0;
- }
- len = 0;
- feature_len = 0;
- }
-
- TensorDim(unsigned int b, unsigned int c, unsigned int h, unsigned int w) :
- TensorDim() {
- setTensorDim(0, b);
- setTensorDim(1, c);
- setTensorDim(2, h);
- setTensorDim(3, w);
- feature_len = c * h * w;
- len = b * feature_len;
- }
-
- TensorDim(const TensorDim &rhs) = default;
-
- TensorDim(const std::string &shape);
-
- ~TensorDim(){};
-
- /**
- * @brief Move constructor of Conv 2D Layer.
- * @param[in] Conv2dLayer &&
- */
- TensorDim(TensorDim &&rhs) noexcept = default;
-
- /**
- * @brief Move assignment operator.
- * @parma[in] rhs Optimizer to be moved.
- */
- TensorDim &operator=(TensorDim &&rhs) noexcept;
-
- /**
- * @brief swap variable of Conv2D Layer
- * @parma[out] lhs Optimizer
- * @parma[in] rhs Optimizer
- */
- friend void swap(TensorDim &lhs, TensorDim &rhs) noexcept {
- std::swap_ranges(std::begin(lhs.dim), std::begin(lhs.dim) + MAXDIM,
- std::begin(rhs.dim));
- std::swap(lhs.len, rhs.len);
- std::swap(lhs.feature_len, rhs.feature_len);
- }
-
- unsigned int batch() const { return dim[0]; };
- unsigned int channel() const { return dim[1]; };
- unsigned int height() const { return dim[2]; };
- unsigned int width() const { return dim[3]; };
- unsigned int getDataLen() const { return len; };
- unsigned int getFeatureLen() const { return feature_len; };
-
- void resetLen();
- void batch(unsigned int b) { setTensorDim(0, b); }
- void channel(unsigned int c) { setTensorDim(1, c); }
- void height(unsigned int h) { setTensorDim(2, h); }
- void width(unsigned int w) { setTensorDim(3, w); }
-
- const unsigned int *getDim() const { return dim; }
- unsigned int getNumDim() const { return MAXDIM; }
-
- const unsigned int getTensorDim(unsigned int idx) const;
- void setTensorDim(unsigned int idx, unsigned int value);
- int setTensorDim(const std::string &input_shape);
-
- TensorDim &operator=(const TensorDim &rhs);
- bool operator==(const TensorDim &rhs) const;
- bool operator!=(const TensorDim &rhs) const { return !(*this == rhs); }
- bool isEmpty() const { return len == 0; }
- unsigned int rank() const;
-
- unsigned int &operator[](const unsigned int index);
- const unsigned int &operator[](const unsigned int index) const;
-
- /**
- * @brief Calculate standard strides
- *
- * @return std::array <int, MAXDIM>
- */
- std::array<unsigned int, MAXDIM> computeStrides() const {
- return {dim[1] * dim[2] * dim[3], dim[2] * dim[3], dim[3], 1};
- }
-
- /**
- * @brief reverse the dimensions inplace
- */
- void reverse();
-
-private:
- unsigned int dim[MAXDIM];
- unsigned int len;
- unsigned int feature_len;
-};
-
-std::ostream &operator<<(std::ostream &out, TensorDim const &d);
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __TENSOR_DIM_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file tflite_layer.h
- * @date 3 November 2020
- * @brief This is class to encapsulate tflite as a layer of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __TENSORFLOW_LITE_H__
-#define __TENSORFLOW_LITE_H__
-#ifdef __cplusplus
-
-#include <layer_internal.h>
-#include <tensor.h>
-
-#include <tensorflow/contrib/lite/interpreter.h>
-#include <tensorflow/contrib/lite/kernels/register.h>
-#include <tensorflow/contrib/lite/model.h>
-
-namespace nntrainer {
-
-/**
- * @class TfLiteLayer
- * @brief Tensorflow Lite layer
- */
-class TfLiteLayer : public Layer {
-public:
- /**
- * @brief Constructor of NNStreamer Layer
- */
- TfLiteLayer(std::string model = "") :
- Layer(LayerType::LAYER_BACKBONE_TFLITE),
- modelfile(model),
- interpreter(nullptr),
- model(nullptr) {
- trainable = false;
- }
-
- /**
- * @brief Destructor of NNStreamer Layer
- */
- ~TfLiteLayer() = default;
-
- /**
- * @copydoc Layer::forwarding(sharedConstTensors in)
- */
- sharedConstTensors forwarding(sharedConstTensors in);
-
- /**
- * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
- */
- sharedConstTensors backwarding(sharedConstTensors in, int iteration);
-
- /**
- * @copydoc Layer::copy(std::shared_ptr<layer> l)
- */
- void copy(std::shared_ptr<Layer> l);
-
- /**
- * @copydoc Layer::initialize()
- */
- int initialize();
-
- /**
- * @copydoc Layer::setTrainable(bool train)
- */
- void setTrainable(bool train);
-
- /**
- * @brief get the base name for the layer
- * @retval base name of the layer
- */
- std::string getBaseName() { return "BackboneTFLite"; };
-
- using Layer::setProperty;
-
- /**
- * @copydoc Layer::setProperty(const PropertyType type, const std::string
- * &value)
- */
- void setProperty(const PropertyType type, const std::string &value = "");
-
-private:
- std::string modelfile;
- std::unique_ptr<tflite::Interpreter> interpreter;
- std::unique_ptr<tflite::FlatBufferModel> model;
-
- void setDimensions(const std::vector<int> &tensor_idx_list,
- std::vector<TensorDim> &dim, bool is_output);
-};
-
-} // namespace nntrainer
-
-#endif /* __cplusplus */
-#endif /* __TENSORFLOW_LITE_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file util_func.h
- * @date 08 April 2020
- * @brief This is collection of math functions.
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __UTIL_FUNC_H__
-#define __UTIL_FUNC_H__
-#ifdef __cplusplus
-
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @brief get the seed
- * @retVal seed
- */
-unsigned int getSeed();
-
-/**
- * @brief random function
- */
-float random();
-
-/**
- * @brief sqrt function for float type
- * @param[in] x float
- */
-float sqrtFloat(float x);
-
-double sqrtDouble(double x);
-
-/**
- * @brief log function for float type
- * @param[in] x float
- */
-float logFloat(float x);
-
-/**
- * @brief exp function for float type
- * @param[in] x float
- */
-float exp_util(float x);
-
-/**
- * @brief apply padding
- * @param[in] batch batch index
- * @param[in] x input
- * @param[in] padding 2D padding size
- * @retVal Tensor output tensor with batch size is 1 for batch index
- */
-Tensor zero_pad(int batch, Tensor const &in, unsigned int const *padding);
-
-/**
- * @brief strip padding
- * @param[in] x input
- * @param[in] padding 2D padding size
- * @retVal Tensor output tensor without padding
- */
-Tensor strip_pad(Tensor const &in, unsigned int const *padding);
-
-/**
- * @brief rotate 180 dgree
- * @param[in] in input Tensor
- * @retVal Tensor rotated tensor (180 degree)
- */
-Tensor rotate_180(Tensor in);
-
-/**
- * @brief Check Existance of File
- * @param[in] file path of the file to be checked
- * @returns true if file exists, else false
- */
-bool isFileExist(std::string file);
-
-} /* namespace nntrainer */
-
-#endif /* __cplusplus */
-#endif /* __UTIL_FUNC_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0-only
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file weight.h
- * @date 22 September 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Weight Class for Neural Network
- *
- */
-
-#ifndef __WEIGHT_H__
-#define __WEIGHT_H__
-
-#include <tensor.h>
-
-namespace nntrainer {
-
-/**
- * @brief Enumeration of Weight Decay type
- */
-enum class WeightRegularizerType {
- l2norm, /** L2 norm regularizer */
- unknown /** Unknown */
-};
-
-/**
- * @brief Enumeration of Weight Initialization Type
- */
-enum class WeightInitializer {
- WEIGHT_ZEROS, /** Zero initialization */
- WEIGHT_ONES, /** One initialization */
- WEIGHT_LECUN_NORMAL, /** LeCun normal initialization */
- WEIGHT_LECUN_UNIFORM, /** uniform initialization */
- WEIGHT_XAVIER_NORMAL, /** Xavier normal initialization */
- WEIGHT_XAVIER_UNIFORM, /** Xavier uniform initialization */
- WEIGHT_HE_NORMAL, /** He normal initialization */
- WEIGHT_HE_UNIFORM, /** He uniform initialization */
- WEIGHT_UNKNOWN /** Unknown */
-};
-
-/**
- * @class Weight
- * @brief Weight with gradient, and its corresponding trainable property
- */
-class Weight {
-
- /** Declare layers as friend to get variable/gradient reference */
- friend class Layer;
- friend class Conv2DLayer;
- friend class FullyConnectedLayer;
- friend class BatchNormalizationLayer;
-
- /** Declare opitmizer as friend to get variable/gradient reference */
- friend class Optimizer;
- friend class SGD;
- friend class Adam;
-
-public:
- /**
- * @brief Weight default constructor
- */
- Weight() : initializer(WeightInitializer::WEIGHT_UNKNOWN), trainable(false) {}
-
- /**
- * @brief Construct a new Weight object
- *
- * @param dim Variable and gradient tensor dimension
- * @param init Initializer for the tensor
- * @param train If the variable is trainable
- * @param name Name for this weight
- */
- Weight(
- const TensorDim &dim,
- const WeightInitializer init = WeightInitializer::WEIGHT_XAVIER_UNIFORM,
- bool train = true, std::string name = "");
-
- /**
- * @brief Allocate and initialize the variable
- *
- * @param dim Dimension for the variable
- */
- void initializeVar(const TensorDim &dim);
-
- /**
- * @brief Swap for weight
- *
- * @param lhs Swap to
- * @param rhs Swap from
- * @note Only swap gradient if trainable
- */
- friend void swap(Weight &lhs, Weight &rhs) noexcept {
- using std::swap;
-
- swap(lhs.var, rhs.var);
- swap(lhs.initializer, rhs.initializer);
- swap(lhs.trainable, rhs.trainable);
- swap(lhs.grad, rhs.grad);
- swap(lhs.name, rhs.name);
- }
-
- /**
- * @brief Copy constructor for weight
- *
- * @param rhs weight to construct from
- */
- Weight(const Weight &rhs);
-
- /**
- * @brief Move constructor for weight
- *
- * @param rhs weight to construct from
- */
- Weight(Weight &&rhs) = default;
-
- /**
- * @brief copy assigment
- *
- * @param rhs copy from
- * @return Weight& Updated weight
- */
- Weight &operator=(const Weight &rhs);
-
- /**
- * @brief move assignment
- *
- * @param rhs move from
- * @return Weight& Updated weight
- */
- Weight &operator=(Weight &&rhs) = default;
-
- /**
- * @brief Get the TensorDim
- *
- * @return TensorDim Dimension
- */
- TensorDim getDim() { return var.getDim(); }
-
- /**
- * @brief Get if the weight is trainable
- *
- * @return true if trainable
- * @return false is not trainable
- */
- bool getTrainable() { return trainable; }
-
- /**
- * @brief Get the name of the weight
- *
- * @return std::string name
- */
- std::string getName() { return name; }
-
- /**
- * @brief Get the variable tensor (by name)
- *
- * @return Tensor Variable tensor
- */
- Tensor getVariable() { return var; }
-
- /**
- * @brief Get the Gradient tensor (by name)
- *
- * @return Tensor Gradient tensor
- */
- Tensor getGradient() { return grad; }
-
-private:
- /**
- * @brief Get the variable tensor (by reference)
- *
- * @return Tensor Variable tensor
- */
- Tensor &getVariableRef() { return var; }
-
- /**
- * @brief Get the Gradient tensor (by reference)
- *
- * @return Tensor Gradient tensor
- */
- Tensor &getGradientRef() { return grad; }
-
- Tensor var; /**< variable to be updated and used */
- Tensor grad; /**< gradient for the variable */
- WeightInitializer initializer; /**< initializer for this variable */
- bool trainable; /**< if this variable is trainable */
- std::string name; /**< name of the parameter */
-};
-
-} // namespace nntrainer
-
-#endif /** __WEIGHT_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jihoon Lee <jhoon.it.lee@samsung.com>
+ *
+ * @file activation_layer.cpp
+ * @date 17 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jihoon Lee <jhoon.it.lee@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Activation Layer Class for Neural Network
+ *
+ */
+
+#include <algorithm>
+#include <cmath>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <vector>
+
+#include <activation_layer.h>
+#include <blas_interface.h>
+#include <layer_internal.h>
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <optimizer_internal.h>
+#include <parse_util.h>
+#include <tensor.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Initialize the layer
+ *
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+int ActivationLayer::initialize() {
+
+ output_dim = input_dim;
+
+ return ML_ERROR_NONE;
+}
+
+sharedConstTensors ActivationLayer::forwarding(sharedConstTensors in) {
+ input = *in[0];
+ /// @note @a _act_fn is expected to work out of place and not modify @a input
+ hidden = _act_fn(input);
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors ActivationLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ Tensor deriv = *derivative[0];
+ Tensor ret;
+ if (activation_type == ActivationType::ACT_SOFTMAX)
+ ret = _act_prime_fn(hidden, deriv);
+ else
+ ret = _act_prime_fn(input, deriv);
+
+ return {MAKE_SHARED_TENSOR(std::move(ret))};
+}
+
+int ActivationLayer::setActivation(
+ std::function<Tensor(Tensor const &)> const &activation_fn,
+ std::function<Tensor(Tensor const &, Tensor const &)> const
+ &activation_prime_fn) {
+ _act_fn = activation_fn;
+ _act_prime_fn = activation_prime_fn;
+
+ return ML_ERROR_NONE;
+}
+
+int ActivationLayer::setActivation(
+ std::function<Tensor(Tensor const &)> const &activation_fn,
+ std::function<Tensor(Tensor const &)> const &activation_prime_fn) {
+ _act_fn = activation_fn;
+ _act_prime_fn = [activation_prime_fn](Tensor const &x,
+ Tensor const &derivative) {
+ return derivative.multiply(activation_prime_fn(x));
+ };
+
+ return ML_ERROR_NONE;
+}
+
+int ActivationLayer::setActivation(
+ std::function<float(float const)> const &activation_fn,
+ std::function<float(float const)> const &activation_prime_fn) {
+ _act_fn = [activation_fn](Tensor const &x) { return x.apply(activation_fn); };
+ _act_prime_fn = [activation_prime_fn](Tensor const &x,
+ Tensor const &derivative) {
+ return derivative.multiply(x.apply(activation_prime_fn));
+ };
+
+ return ML_ERROR_NONE;
+}
+
+/**
+ * @brief setActivation by preset ActivationType
+ *
+ * @param[in] ActivationType ActivationType ActivationType to be set
+ */
+void ActivationLayer::setActivation(ActivationType acti_type) {
+ Layer::setActivation(acti_type);
+
+ switch (acti_type) {
+ case ActivationType::ACT_TANH:
+ this->setActivation(tanhFloat, tanhPrime);
+ break;
+ case ActivationType::ACT_SIGMOID:
+ this->setActivation(sigmoid, sigmoidPrime);
+ break;
+ case ActivationType::ACT_SOFTMAX:
+ this->setActivation(softmax, softmaxPrime);
+ break;
+ case ActivationType::ACT_RELU:
+ this->setActivation(relu, reluPrime);
+ break;
+ case ActivationType::ACT_NONE:
+ this->setActivation(no_op, no_op_prime);
+ break;
+ case ActivationType::ACT_UNKNOWN:
+ default:
+ throw std::runtime_error("Error: Not Supported Activation Type");
+ }
+}
+
+Tensor ActivationLayer::softmax(Tensor const &t) {
+ /**
+ * shiftx_logit = logit - max_batch(logit)
+ * softmax = exp(shiftx_logit) / (sum(exp(shiftx_logit)))
+ */
+ unsigned int batch = t.batch();
+ float *dp;
+ float *rp;
+
+ Tensor divisor = t.clone();
+
+ dp = divisor.getData();
+ unsigned int feat_len = t.getDim().getFeatureLen();
+
+ for (unsigned int k = 0; k < batch; k++) {
+ int index = k * feat_len;
+ // find max and subtract it
+ float m = *std::max_element(dp + index, dp + index + feat_len);
+
+ Tensor tmp = Tensor(1, 1, 1, feat_len);
+ tmp.setValue(m);
+ saxpy(feat_len, -1, tmp.getData(), 1, dp + index, 1);
+ }
+
+ // take exp
+ Tensor result = divisor.apply(exp_util);
+ rp = result.getData();
+ // take sum over batch
+ Tensor sum = result.sum_by_batch();
+
+ for (unsigned int k = 0; k < batch; k++) {
+ int index = k * feat_len;
+ std::transform(rp + index, rp + index + feat_len, rp + index,
+ std::bind(std::divides<float>(), std::placeholders::_1,
+ sum.getValue(k, 0, 0, 0)));
+ }
+
+ return result;
+}
+
+Tensor ActivationLayer::softmaxPrime(Tensor const &x,
+ Tensor const &derivative) {
+ unsigned int batch = x.batch();
+ unsigned int channel = x.channel();
+ unsigned int height = x.height();
+ unsigned int width = x.width();
+ bool is_derivative = true;
+
+ Tensor PI = Tensor(x.getDim());
+
+ const float *xp = x.getData();
+ const float *d = derivative.getData();
+ float *pp = PI.getData();
+
+ /** @todo update default tensorDim to be 0 and not 1 */
+ if (derivative.getDim() == TensorDim()) {
+ is_derivative = false;
+ }
+
+ for (unsigned int k = 0; k < batch; ++k) {
+ int K = k * channel * height * width;
+ for (unsigned int c = 0; c < channel; ++c) {
+ int C = K + c * height * width;
+ for (unsigned int i = 0; i < height; ++i) {
+ int I = C + i * width;
+ for (unsigned int j = 0; j < width; ++j) {
+ float sum = 0.0f;
+ for (unsigned int l = 0; l < width; ++l) {
+ float val;
+ if (j == l) {
+ val = xp[I + l] * (1.0f - xp[I + j]);
+ } else {
+ val = -xp[I + l] * xp[I + j];
+ }
+ if (is_derivative)
+ val *= d[I + l];
+ sum += val;
+ }
+ pp[I + j] = sum;
+ }
+ }
+ }
+ }
+ return PI;
+}
+
+float ActivationLayer::sigmoid(float x) { return 1.0f / (1.0f + exp_util(-x)); }
+
+float ActivationLayer::sigmoidPrime(float x) {
+ float sprime = sigmoid(x);
+ return sprime * (1.0f - sprime);
+}
+
+float ActivationLayer::tanhFloat(float x) { return (float)tanh(x); }
+
+float ActivationLayer::tanhPrime(float x) {
+ float th = (float)tanh(x);
+ return 1.0f - th * th;
+}
+
+float ActivationLayer::relu(float x) {
+ if (x <= 0.0f) {
+ return 0.0f;
+ } else {
+ return x;
+ }
+}
+
+float ActivationLayer::reluPrime(float x) {
+ if (x <= 0.0f) {
+ return 0.0f;
+ } else {
+ return 1.0f;
+ }
+}
+
+float ActivationLayer::no_op(float x) { return x; }
+
+float ActivationLayer::no_op_prime(float x) { return 1.0f; }
+}; // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jihoon Lee <jhoon.it.lee@samsung.com>
+ *
+ * @file activation_layer.h
+ * @date 17 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jihoon Lee <jhoon.it.lee@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Activation Layer Class for Neural Network
+ *
+ */
+
+#ifndef __ACTIVATION_LAYER_H__
+#define __ACTIVATION_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class Activation Layer
+ * @brief Activation Layer
+ */
+class ActivationLayer : public Layer {
+
+public:
+ /**
+ * @brief Constructor of Activation Layer
+ */
+ template <typename... Args>
+ ActivationLayer(ActivationType at = ActivationType::ACT_NONE, Args... args) :
+ Layer(LayerType::LAYER_ACTIVATION, args...) {
+ setActivation(at);
+ }
+
+ /**
+ * @brief Destructor of Activation Layer
+ */
+ ~ActivationLayer(){};
+
+ /**
+ * @brief Initialize the layer
+ *
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief Read Activation layer params. This is essentially noops for now.
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file){/* noop */};
+
+ /**
+ * @brief Save Activation layer params. This is essentially noops for now.
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file){/* noop */};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief setActivation by preset ActivationType
+ *
+ * @param[in] ActivationTypeeActivationTypeeActivationTypeet
+ */
+ void setActivation(ActivationType acti_type);
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Activation"; };
+
+ /**
+ * @brief Calculate softmax for Tensor Type
+ * @param[in] t Tensor
+ * @retval Tensor
+ */
+ static Tensor softmax(Tensor const &x);
+
+ /**
+ * @brief derivative softmax function for Tensor Type
+ * @param[in] x Tensor
+ * @retVal Tensor
+ */
+ static Tensor softmaxPrime(Tensor const &x,
+ Tensor const &derivative = Tensor());
+
+ /**
+ * @brief sigmoid activation function
+ * @param[in] x input
+ */
+ static float sigmoid(float x);
+
+ /**
+ * @brief derivative sigmoid function
+ * @param[in] x input
+ */
+ static float sigmoidPrime(float x);
+
+ /**
+ * @brief tanh function for float type
+ * @param[in] x input
+ */
+ static float tanhFloat(float x);
+
+ /**
+ * @brief derivative tanh function
+ * @param[in] x input
+ */
+ static float tanhPrime(float x);
+
+ /**
+ * @brief relu activation function
+ * @param[in] x input
+ */
+ static float relu(float x);
+
+ /**
+ * @brief derivative relu function
+ * @param[in] x input
+ */
+ static float reluPrime(float x);
+
+ /**
+ * @brief no_op function
+ * @param[in] x input
+ */
+ static float no_op(float x);
+
+ /**
+ * @brief no_op function
+ * @param[in] x input
+ */
+ static float no_op_prime(float x);
+
+private:
+ std::function<Tensor(Tensor const &)> _act_fn;
+ std::function<Tensor(Tensor const &, Tensor const &)> _act_prime_fn;
+
+ /**
+ * @brief setActivation by custom activation function
+ * @note apply derivative as this activation_prime_fn does not utilize
+ * derivative
+ * @param[in] std::function<Tensor(Tensor const &)> activation_fn activation
+ * function to be used
+ * @param[in] std::function<Tensor(Tensor const &)> activation_prime_fn
+ * activation_prime_function to be used
+ * @retval #ML_ERROR_NONE when successful
+ */
+ int setActivation(
+ std::function<Tensor(Tensor const &)> const &activation_fn,
+ std::function<Tensor(Tensor const &)> const &activation_prime_fn);
+
+ /**
+ * @brief setActivation by custom activation function
+ * @note derivative not applied here as this activation_prime_fn applies
+ * derivative itself
+ * @param[in] std::function<Tensor(Tensor const &)> activation_fn activation
+ * function to be used
+ * @param[in] std::function<Tensor(Tensor const &, Tensor const &)>
+ * activation_prime_fn activation_prime_function to be used
+ * @retval #ML_ERROR_NONE when successful
+ */
+ int setActivation(std::function<Tensor(Tensor const &)> const &activation_fn,
+ std::function<Tensor(Tensor const &, Tensor const &)> const
+ &activation_prime_fn);
+
+ /**
+ * @brief setActivation by custom activation function
+ * @note apply derivative as this activation_prime_fn does not utilize
+ * derivative
+ * @param[in] std::function<float(float const &)> activation_fn activation
+ * function to be used
+ * @param[in] std::function<float(float const &)> activation_prime_fn
+ * activation_prime_function to be used
+ * @retval #ML_ERROR_NONE when successful
+ */
+ int setActivation(
+ std::function<float(float const)> const &activation_fn,
+ std::function<float(float const)> const &activation_prime_fn);
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __ACTIVATION_LAYER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file addition_layer.cpp
+ * @date 30 July 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Addition Layer Class for Neural Network
+ *
+ */
+
+#include <addition_layer.h>
+#include <layer_internal.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int AdditionLayer::initialize() {
+ int status = ML_ERROR_NONE;
+ if (num_inputs == 0) {
+ ml_loge("Error: number of inputs are not initialized");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ for (unsigned int idx = 0; idx < num_inputs; ++idx) {
+ if (input_dim[idx].getDataLen() == 1) {
+ ml_logw("Warning: the length of previous layer dimension is one");
+ }
+ }
+
+ /** input dimension indicates the dimension for all the inputs to follow */
+ output_dim[0] = input_dim[0];
+
+ return status;
+}
+
+sharedConstTensors AdditionLayer::forwarding(sharedConstTensors in) {
+ hidden = Tensor(input_dim[0]);
+ hidden.setZero();
+
+ TensorDim &in_dim = input_dim[0];
+
+ for (unsigned int idx = 0; idx < num_inputs; ++idx) {
+ if (in_dim != in[idx]->getDim())
+ throw std::runtime_error("Error: addition layer requires same "
+ "shape from all input layers");
+ hidden.add_i(*in[idx]);
+ }
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors AdditionLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+
+ sharedConstTensors ret;
+ for (unsigned int i = 0; i < num_inputs; ++i) {
+ sharedTensor t =
+ std::shared_ptr<Tensor>(new Tensor(), std::default_delete<Tensor[]>());
+ *t = *derivative[0];
+ ret.push_back(t);
+ }
+
+ return ret;
+}
+
+void AdditionLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::num_inputs: {
+ if (!value.empty()) {
+ status = setUint(num_inputs, value);
+ throw_status(status);
+ if (num_inputs < 1)
+ throw std::invalid_argument("Minimum number of inputs must be 1");
+ }
+ } break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file addition_layer.h
+ * @date 30 July 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Addition Layer Class for Neural Network
+ *
+ */
+
+#ifndef __ADDITION_LAYER_H__
+#define __ADDITION_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class Addition Layer
+ * @brief Addition Layer
+ */
+class AdditionLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Addition Layer
+ */
+ template <typename... Args>
+ AdditionLayer(unsigned int num_inputs_ = 0, Args... args) :
+ Layer(LayerType::LAYER_ADDITION, args...) {
+ num_inputs = num_inputs_;
+ }
+
+ /**
+ * @brief Destructor of Addition Layer
+ */
+ ~AdditionLayer(){};
+
+ /**
+ * @brief Move constructor of AdditionLayer.
+ * @param[in] AdditionLayer &&
+ */
+ AdditionLayer(AdditionLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs AdditionLayer to be moved.
+ */
+ AdditionLayer &operator=(AdditionLayer &&rhs) = default;
+
+ /**
+ * @brief initialize layer
+ * @param[in] last last layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file){};
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file){};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Addition"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __ADDITION_LAYER_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file bn_layer.cpp
+ * @date 14 May 2020
+ * @brief This is Batch Normalization Layer Class for Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <assert.h>
+#include <bn_layer.h>
+#include <layer_internal.h>
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+enum class BNParams { mu, var, gamma, beta };
+
+/// @todo add multiple axis support
+int BatchNormalizationLayer::initialize() {
+ int status = ML_ERROR_NONE;
+
+ if (num_inputs != 1) {
+ throw std::invalid_argument(
+ "Only one input is allowed for batch normalization layer");
+ }
+
+ output_dim[0] = input_dim[0];
+ TensorDim dim;
+
+ /// @note this logic cannot tell channel is actually 1 or it is just not used.
+ if (axis == -1)
+ axis = input_dim[0].channel() > 1 ? 1 : 3;
+
+ dim.setTensorDim(axis, input_dim[0].getTensorDim(axis));
+
+ for (int i = 0; i < 4; ++i) {
+ if (axis != i)
+ axes_to_reduce.push_back(i);
+ }
+
+ setNumWeights(4);
+ weightAt(0) =
+ std::move(Weight(dim, initializers[static_cast<int>(BNParams::mu)], false,
+ "BN:moving_mean"));
+ ///@todo shift var to std to save computation
+ weightAt(1) =
+ std::move(Weight(dim, initializers[static_cast<int>(BNParams::var)], false,
+ "BN:moving_variance"));
+ weightAt(2) = std::move(Weight(
+ dim, initializers[static_cast<int>(BNParams::gamma)], true, "BN:gamma"));
+ weightAt(3) = std::move(Weight(
+ dim, initializers[static_cast<int>(BNParams::beta)], true, "BN:beta"));
+
+ return status;
+}
+
+void BatchNormalizationLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+ switch (type) {
+ case PropertyType::epsilon:
+ if (!value.empty()) {
+ status = setFloat(epsilon, value);
+ throw_status(status);
+ }
+ break;
+ case PropertyType::moving_mean_initializer:
+ if (!value.empty()) {
+ initializers[static_cast<int>(BNParams::mu)] =
+ (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
+ }
+ break;
+ case PropertyType::moving_variance_initializer:
+ if (!value.empty()) {
+ initializers[static_cast<int>(BNParams::var)] =
+ (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
+ }
+ break;
+ case PropertyType::beta_initializer:
+ if (!value.empty()) {
+ initializers[static_cast<int>(BNParams::beta)] =
+ (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
+ }
+ break;
+ case PropertyType::gamma_initializer:
+ if (!value.empty()) {
+ initializers[static_cast<int>(BNParams::gamma)] =
+ (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
+ }
+ break;
+ case PropertyType::momentum:
+ if (!value.empty()) {
+ status = setFloat(momentum, value);
+ throw_status(status);
+ }
+ break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+sharedConstTensors BatchNormalizationLayer::forwarding(sharedConstTensors in) {
+ Tensor &mu = weightAt(static_cast<int>(BNParams::mu)).getVariableRef();
+ Tensor &var = weightAt(static_cast<int>(BNParams::var)).getVariableRef();
+ Tensor &gamma = weightAt(static_cast<int>(BNParams::gamma)).getVariableRef();
+ Tensor &beta = weightAt(static_cast<int>(BNParams::beta)).getVariableRef();
+
+ input = *in[0];
+ /// @todo change trainable #524
+ if (trainable) {
+ Tensor cmu = input.average(axes_to_reduce);
+ deviation = input.subtract(cmu);
+
+ cvar = deviation.pow(2.0f).average(axes_to_reduce);
+
+ mu.multiply_i(momentum);
+ mu.add_i(cmu, 1 - momentum);
+ var.multiply_i(momentum);
+ var.add_i(cvar, 1 - momentum);
+
+ cvar.add_i(epsilon);
+
+ invstd = cvar.pow(-0.5f);
+ this->x_normalized = deviation.multiply(invstd);
+ this->hidden = x_normalized.multiply(gamma);
+ this->hidden.add_i(beta);
+ } else {
+ deviation = input.subtract(mu);
+ this->x_normalized = deviation.divide(var.add(epsilon).pow(0.5f));
+ this->hidden = x_normalized.multiply(gamma);
+ this->hidden.add(beta);
+ }
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors
+BatchNormalizationLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ Tensor &gamma = weightAt(static_cast<int>(BNParams::gamma)).getVariableRef();
+ Tensor &dgamma = weightAt(static_cast<int>(BNParams::gamma)).getGradientRef();
+ Tensor &dbeta = weightAt(static_cast<int>(BNParams::beta)).getGradientRef();
+ Tensor dx_normalized;
+
+ Tensor deriv = *derivative[0];
+
+ int N = 1;
+
+ for (auto &axis : axes_to_reduce) {
+ N *= input_dim[0].getTensorDim(axis);
+ }
+
+ dbeta = deriv.sum(axes_to_reduce);
+ dgamma = deviation.multiply(invstd).multiply(deriv).sum(axes_to_reduce);
+
+ Tensor dx_1 = gamma.multiply(invstd);
+ Tensor dx_2 = deriv.multiply(N);
+ dx_2.subtract_i(deriv.sum(axes_to_reduce));
+ dx_2.subtract_i(deviation.divide(cvar).multiply(
+ deviation.multiply(deriv).sum(axes_to_reduce)));
+
+ Tensor dx = dx_2.multiply(dx_1);
+ dx.divide_i(N);
+
+ opt->apply_gradients(weight_list, num_weights, iteration);
+
+ return {MAKE_SHARED_TENSOR(std::move(dx))};
+}
+
+void BatchNormalizationLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<BatchNormalizationLayer> from =
+ std::static_pointer_cast<BatchNormalizationLayer>(l);
+ this->cvar.copy(from->cvar);
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file bn_layer.h
+ * @date 14 May 2020
+ * @brief This is Batch Normalization Layer Class of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __BN_LAYER_H__
+#define __BN_LAYER_H__
+#ifdef __cplusplus
+
+#include <array>
+#include <functional>
+#include <vector>
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class BatchNormalizationLayer
+ * @brief Batch Noramlization Layer
+ */
+class BatchNormalizationLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Batch Noramlization Layer
+ */
+ template <typename... Args>
+ BatchNormalizationLayer(
+ int axis = -1, float momentum = 0.99, float epsilon = 0.001,
+ WeightInitializer moving_mean_initializer = WeightInitializer::WEIGHT_ZEROS,
+ WeightInitializer moving_variance_initializer =
+ WeightInitializer::WEIGHT_ZEROS,
+ WeightInitializer gamma_initializer = WeightInitializer::WEIGHT_ONES,
+ WeightInitializer beta_initializer = WeightInitializer::WEIGHT_ONES,
+ Args... args) :
+ Layer(LayerType::LAYER_BN, args...),
+ epsilon(epsilon),
+ momentum(momentum),
+ axis(axis),
+ initializers{moving_variance_initializer, moving_variance_initializer,
+ gamma_initializer, beta_initializer} {}
+
+ /**
+ * @brief Destructor of BatchNormalizationLayer
+ */
+ ~BatchNormalizationLayer() {}
+
+ /**
+ * @brief Move constructor of Pooling 2D Layer.
+ * @param[in] BatchNormalization &&
+ */
+ BatchNormalizationLayer(BatchNormalizationLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs BatchNormalizationLayer to be moved.
+ */
+ BatchNormalizationLayer &operator=(BatchNormalizationLayer &&rhs) = default;
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /**
+ * @brief initialize layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "BatchNormalization"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ Tensor cvar; /**< training variance saved in bn_layer::forwarding and used in
+ bn_layer::backwarding */
+ Tensor invstd; /**< inversed training std for backward pass */
+
+ Tensor deviation; /**< (input - current_average) */
+
+ Tensor x_normalized; /**< normalized axis saved for backwarding */
+ float epsilon; /**< epsilon */
+ float momentum; /**< momentum */
+ int axis; /**< Target axis, axis inferred at initialize when -1 */
+
+ std::vector<unsigned int> axes_to_reduce; /**< target axes to reduce */
+ std::array<WeightInitializer, 4> initializers; /**< weight initializers */
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __BN_LAYER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file concat_layer.cpp
+ * @date 27 Oct 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Concat Layer Class for Neural Network
+ *
+ */
+
+#include <concat_layer.h>
+#include <cstring>
+#include <layer_internal.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int ConcatLayer::initialize() {
+ int status = ML_ERROR_NONE;
+ unsigned int channel = 0;
+
+ if (num_inputs == 0) {
+ ml_loge("Error: number of inputs are not initialized");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ const TensorDim &d = input_dim[0];
+ channel += d.channel();
+ for (unsigned int idx = 1; idx < num_inputs; ++idx) {
+ const TensorDim &dim = input_dim[idx];
+
+ for (unsigned int i = 2; i < d.rank(); ++i) {
+ if (d[i] != dim[i])
+ throw std::runtime_error("Error: concat layer requires same "
+ "shape from all input layers");
+ }
+ channel += input_dim[idx].channel();
+ }
+
+ output_dim[0] = input_dim[0];
+ output_dim[0].channel(channel);
+
+ return status;
+}
+
+sharedConstTensors ConcatLayer::forwarding(sharedConstTensors in) {
+ hidden = Tensor(output_dim[0]);
+
+#ifdef DEBUG
+ const TensorDim &d = in[0]->getDim();
+ channel += d.channel();
+ for (unsigned int idx = 1; idx < num_inputs; ++idx) {
+ const TensorDim &dim = in[idx]->getDim();
+
+ for (unsigned int i = 2; i < d.rank(); ++i) {
+ if (d[i] != dim[i])
+ throw std::runtime_error("Error: concat layer requires same "
+ "shape from all input layers");
+ }
+ channel += input_dim[idx].channel();
+ }
+
+ if (channel != output_dim[0].channel())
+ throw std::runtime_error(
+ "Error: Sum of channel of input layers is not same with output channel");
+#endif
+
+ unsigned int f_size = output_dim[0].getFeatureLen();
+
+ for (unsigned int b = 0; b < input_dim[0].batch(); ++b) {
+ unsigned int position = 0;
+ for (unsigned int idx = 0; idx < num_inputs; ++idx) {
+ TensorDim in_dim = in[idx]->getDim();
+ memcpy(hidden.getAddress(b * f_size + position),
+ in[idx]->getAddress(b * in_dim.getFeatureLen()),
+ in_dim.getFeatureLen() * sizeof(float));
+ position += in_dim.getFeatureLen();
+ }
+ }
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors ConcatLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ sharedConstTensors ret;
+ TensorDim d = derivative[0]->getDim();
+
+ unsigned int position = 0;
+ for (unsigned int idx = 0; idx < num_inputs; ++idx) {
+ TensorDim in_dim = input_dim[idx];
+ sharedTensor t = std::shared_ptr<Tensor>(new Tensor(in_dim),
+ std::default_delete<Tensor>());
+
+ for (unsigned int b = 0; b < in_dim.batch(); ++b) {
+ memcpy(t->getAddress(b * in_dim.getFeatureLen()),
+ derivative[0]->getAddress(b * d.getFeatureLen() + position),
+ in_dim.getFeatureLen() * sizeof(float));
+ }
+ position += in_dim.getFeatureLen();
+
+ ret.push_back(t);
+ }
+
+ return ret;
+}
+
+void ConcatLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::num_inputs: {
+ if (!value.empty()) {
+ status = setUint(num_inputs, value);
+ throw_status(status);
+ if (num_inputs < 1)
+ throw std::invalid_argument("Minimum number of inputs must be 1");
+ }
+ } break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file concat_layer.h
+ * @date 27 Oct 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Concat Layer Class for Neural Network
+ *
+ */
+
+#ifndef __CONCAT_LAYER_H__
+#define __CONCAT_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class Concat Layer
+ * @brief Concat Layer
+ */
+class ConcatLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Concat Layer
+ */
+ template <typename... Args>
+ ConcatLayer(unsigned int num_inputs_ = 0, Args... args) :
+ Layer(LayerType::LAYER_CONCAT, args...) {
+ num_inputs = num_inputs_;
+ }
+
+ /**
+ * @brief Destructor of Concat Layer
+ */
+ ~ConcatLayer(){};
+
+ /**
+ * @brief Move constructor of ConcatLayer.
+ * @param[in] ConcatLayer &&
+ */
+ ConcatLayer(ConcatLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs ConcatLayer to be moved.
+ */
+ ConcatLayer &operator=(ConcatLayer &&rhs) = default;
+
+ /**
+ * @brief initialize layer
+ * @param[in] last last layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file){};
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file){};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Concat"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __CONCAT_LAYER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file conv2d_layer.h
+ * @date 02 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Convolution Layer Class for Neural Network
+ *
+ */
+#include <cstring>
+#include <string>
+
+#include <blas_interface.h>
+#include <conv2d_layer.h>
+#include <layer_internal.h>
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int Conv2DLayer::initialize() {
+ int status = ML_ERROR_NONE;
+
+ if (input_dim.size() != 1 || output_dim.size() != 1) {
+ throw std::invalid_argument("Convolution layer only takes one input");
+ }
+
+ TensorDim &in_dim = input_dim[0];
+ TensorDim &out_dim = output_dim[0];
+
+ if (in_dim.getDataLen() == 1) {
+ ml_logw("Warning: the length of previous layer dimension is one");
+ }
+
+ TensorDim dim =
+ TensorDim(1, in_dim.channel(), kernel_size[0], kernel_size[1]);
+ TensorDim bias_dim = TensorDim(1, 1, 1, 1);
+
+ std::string kernelPrefix = "Conv2d:filter";
+ std::string biasPrefix = "Conv2d:bias";
+ setNumWeights(filter_size * 2);
+
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ /*< @note: order of weight and bias are:
+ w0 w1 w2 ... w3
+ */
+ weightAt(i) = std::move(
+ Weight(dim, weight_initializer, true, kernelPrefix + std::to_string(i)));
+ weightAt(i + filter_size) = std::move(
+ Weight(bias_dim, bias_initializer, true, biasPrefix + std::to_string(i)));
+ }
+
+ // this output_dim should be the same with dimension of hidden
+ out_dim.batch(in_dim.batch());
+ out_dim.channel(filter_size);
+ out_dim.height(
+ (in_dim.height() - kernel_size[0] + 2 * padding[0]) / stride[0] + 1);
+ out_dim.width((in_dim.width() - kernel_size[1] + 2 * padding[1]) / stride[1] +
+ 1);
+
+ return status;
+}
+
+void Conv2DLayer::read(std::ifstream &file) { Layer::read(file); }
+
+void Conv2DLayer::save(std::ofstream &file) { Layer::save(file); }
+
+sharedConstTensors Conv2DLayer::forwarding(sharedConstTensors in) {
+ int status = ML_ERROR_NONE;
+
+ if (num_inputs != 1)
+ throw std::invalid_argument("Convolution layer only takes one input");
+
+ input = *in[0];
+
+ TensorDim &in_dim = input_dim[0];
+ TensorDim &out_dim = output_dim[0];
+
+ if (normalization) {
+ input = input.normalization();
+ }
+
+ if (standardization) {
+ input = input.standardization();
+ }
+
+ TensorDim hidden_dim = output_dim[0];
+ hidden = Tensor(hidden_dim);
+ hidden.setZero();
+
+ /** Calculate Convolution 2D
+ *
+ * This is the 2D Matrix Shape [ height ] x [ width ]
+ * . Height : filter_size
+ * . Width : Input Channel * Kernel_size[0] * Kernel_size[1]
+ *
+ * imKernel
+ * +------|------|------+
+ * |------|------|------|
+ * [filter_size (height)] |------|------|------|
+ * |------|------|------|
+ * +------|------|------+
+ * [Input Channel * Kernel_size[0]
+ * * Kernel_size[1] (width)]
+ *
+ *
+ * After im2Col with channel_mode true (in : input)
+ *
+ * This is the 2D Matrix Shape [ height ] x [ width ]
+ * . Height : Input Channel * Kernel_size[0] * Kernel_size[1]
+ * . Width : output_dim.height * output_dim.width
+ *
+ * +-|-|-|-| |-|-|-|-+
+ * [Input Channel | | | | | | | | | |
+ * * Kernel_size[0] |_|_|_|_| |_|_|_|_|
+ * * Kenel_size[1] | | | | | .... | | | | |
+ * (height)] |_|_|_|_| |_|_|_|_|
+ * | | | | | | | | | |
+ * +_|_|_|_| |_|_|_|_+
+ * [ output_dim.height
+ * * output_dim.width (width) ]
+ *
+ * Output Dimention
+ * -> [Channel ( = filter_size = output_dim.channel )]
+ * x [output_dim.height x output_dim.width]
+ */
+
+ TensorDim kdim(filter_size, in_dim.channel(), kernel_size[0], kernel_size[1]);
+
+ std::vector<float> imkernel(kdim.getFeatureLen() * filter_size);
+
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ Tensor &filters = weightAt(i).getVariableRef();
+ float *d = imkernel.data();
+ memcpy(&d[i * kdim.getFeatureLen()], filters.getData(),
+ kdim.getFeatureLen() * sizeof(float));
+ }
+
+ for (unsigned int b = 0; b < in_dim.batch(); ++b) {
+ std::vector<float> out(out_dim.getFeatureLen());
+ Tensor inSub(TensorDim(1, input.channel(), input.height(), input.width()),
+ input.getAddress(b * input.getDim().getFeatureLen()));
+
+ status = conv2d_gemm(imkernel.data(), kdim, inSub, out_dim, stride, padding,
+ out.data(), out.size(), true);
+ if (status != ML_ERROR_NONE)
+ throw std::runtime_error("Forwarding Convolution failed.");
+ memcpy(hidden.getAddress(b * hidden.getDim().getFeatureLen()), out.data(),
+ out.size() * sizeof(float));
+
+ for (unsigned int i = 0; i < filter_size; i++) {
+ Tensor &bias = weightAt(i + filter_size).getVariableRef();
+ Tensor tmp(1, 1, hidden.height(), hidden.width());
+ tmp.setValue(bias.getValue(0, 0, 0, 0));
+ saxpy(hidden.height() * hidden.width(), 1, tmp.getData(), 1,
+ hidden.getAddress(b * hidden.getDim().getFeatureLen() +
+ i * hidden.height() * hidden.width()),
+ 1);
+ }
+ }
+
+ loss = 0.0f;
+ if (weight_regularizer == WeightRegularizerType::l2norm) {
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ Tensor &weight = weightAt(i).getVariableRef();
+ loss += weight_regularizer_constant * 0.5f * (weight.l2norm());
+ }
+ loss /= filter_size;
+ }
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+};
+
+sharedConstTensors Conv2DLayer::backwarding(sharedConstTensors derivatives,
+ int iteration) {
+
+ TensorDim &in_dim = input_dim[0];
+
+ std::array<unsigned int, CONV2D_DIM> same_pad;
+ sharedConstTensor derivative = derivatives[0];
+
+ same_pad[0] = kernel_size[0] - 1;
+ same_pad[1] = kernel_size[1] - 1;
+
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ Tensor &delK = weightAt(i).getGradientRef();
+ Tensor &delBias = weightAt(i + filter_size).getGradientRef();
+ delK.setZero();
+ delBias.setZero();
+ }
+
+ Tensor ret(in_dim.batch(), in_dim.channel(), in_dim.height() + padding[0] * 2,
+ in_dim.width() + padding[1] * 2);
+ ret.setZero();
+
+ /** Calculate DelK
+ *
+ * This is the 2D Matrix Shape [ height ] x [ width ]
+ * . Height : filter_size
+ * . Width : derivative.height * derivative.width
+ *
+ * derivative
+ * +------|------+
+ * |------|------|
+ * [filter_size (height) |------|------|
+ * (=derivative->channel) |------|------|
+ * +------|------+
+ * [derivative->height
+ * * derivative->width (width)]
+ *
+ *
+ * After im2Col with channel_mode false ( in : input )
+ *
+ * This is the 2D Matrix Shape [ height ] x [ width ]
+ * . Height : derivative.height * derivative.width
+ * . Width : input_dim.channel * Kernel_size[0] * Kernel_size[1]
+ *
+ * +-|-|-|-| |-|-|-|-+
+ * | | | | | | | | | |
+ * [derivative->width |_|_|_|_| |_|_|_|_|
+ * * derivative->height | | | | | .... | | | | |
+ * (height)] +_|_|_|_| |_|_|_|_+
+ * [ input_dim.channel * kernel_size[0]
+ * * kernel_size[1] (width) ]
+ *
+ * Output Dimension
+ * -> [ derivative->channel = filter_size (height) ]
+ * x [input_dim.channel * kernel_size[0] * kernel_size[1] (width) ]
+ */
+
+ int status = ML_ERROR_NONE;
+ for (unsigned int b = 0; b < in_dim.batch(); ++b) {
+ std::vector<float> out(kernel_size[0] * kernel_size[1] * in_dim.channel() *
+ filter_size);
+
+ Tensor inSub(
+ TensorDim(1, in_dim.channel(), in_dim.height(), in_dim.width()),
+ input.getAddress(b * input.getDim().getFeatureLen()));
+
+ status = conv2d_gemm(
+ derivative->getAddress(b * derivative->getDim().getFeatureLen()),
+ TensorDim(1, derivative->channel(), derivative->height(),
+ derivative->width()),
+ inSub,
+ TensorDim(1, 1, filter_size,
+ kernel_size[0] * kernel_size[1] * in_dim.channel()),
+ stride, padding, out.data(), out.size(), false);
+ if (status != ML_ERROR_NONE)
+ throw std::runtime_error("Backwarding Convolution failed.");
+
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ Tensor &delK = weightAt(i).getGradientRef();
+ Tensor &delBias = weightAt(i + filter_size).getGradientRef();
+ float *del = delK.getData();
+ unsigned int s = kernel_size[0] * kernel_size[1] * in_dim.channel();
+
+ for (unsigned int k = 0; k < s; ++k) {
+ del[k] += out[i * s + k];
+ }
+
+ float sum = 0.0;
+ for (unsigned int j = 0; j < derivative->height(); ++j) {
+ for (unsigned int k = 0; k < derivative->width(); ++k) {
+ sum += derivative->getValue(b, i, j, k);
+ }
+ }
+ delBias.setValue(0, 0, 0, 0, sum + delBias.getValue(0, 0, 0, 0));
+ }
+ }
+
+ /** Calculate return derivative
+ *
+ * This is the 2D Matrix Shape [ height ] x [ width ]
+ * . Height : filter.channel = input_dim.channel
+ * . Width : filter_size * kernel_size[0] * kernel_size[1]
+ *
+ * kernel
+ * f0 f1 fn
+ * +---|---|---|---|...|---|---+
+ * |---|---|---|---|...|---|---|
+ * [filter.channel(height)] |---|---|---|---|...|---|---|
+ * (=input_dim.channel) |---|---|---|---|...|---|---|
+ * +---|---|---|---|...|---|---+
+ * [ filter_size
+ * * kernel_size[0]
+ * * kernel_size[1] (width) ]
+ *
+ *
+ * After im2Col with channel_mode true ( in : derivative with full padding )
+ *
+ * This is the 2D Matrix Shape [ height ] x [ width ]
+ * . Height : filter_size * kernel_size[0] * kernel_size[1]
+ * . Width : (input_dim.height + padding[0]*2) x (input_dim.width +
+ * padding[1]*2)
+ *
+ * +-|-|-|-| |-|-|-|-+
+ * | | | | | | | | | |
+ * [ filter_size |_|_|_|_| |_|_|_|_|
+ * * kernel_size[0] | | | | | .... | | | | |
+ * * kernel_size[1] |_|_|_|_| |_|_|_|_|
+ * (height) ] | | | | | | | | | |
+ * +_|_|_|_| |_|_|_|_+
+ * [(input_dim.height() + padding[0] *2)
+ * * (input_dim.width() + padding[1] *2)]
+ *
+ * Output Dimension
+ *
+ * -> [ input_dim.chennel (height) ]
+ * x [(input_dim.height() + padding[0]*2)
+ * *(input_dim.width() + padding[1]*2) (width)]
+ */
+
+ TensorDim kdim(ret.channel(), filter_size, kernel_size[0], kernel_size[1]);
+
+ std::vector<float> imkernel(kdim.getDataLen());
+
+ unsigned int count = 0;
+ float *d = imkernel.data();
+
+ for (unsigned int j = 0; j < ret.channel(); ++j) {
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ Tensor &filters = weightAt(i).getVariableRef();
+ for (unsigned int k = 0; k < kernel_size[0] * kernel_size[1]; ++k) {
+ d[count++] = filters.getData()[j * kernel_size[0] * kernel_size[1] + k];
+ }
+ }
+ }
+
+ TensorDim input_dim_padded(1, in_dim.channel(),
+ in_dim.height() + padding[0] * 2,
+ in_dim.width() + padding[1] * 2);
+
+ for (unsigned int b = 0; b < in_dim.batch(); ++b) {
+ Tensor inSub(
+ TensorDim(1, derivative->channel(), derivative->height(),
+ derivative->width()),
+ derivative->getAddress(b * derivative->getDim().getFeatureLen()));
+
+ status =
+ conv2d_gemm(imkernel.data(), kdim, inSub, input_dim_padded, stride,
+ same_pad, ret.getAddress(b * ret.getDim().getFeatureLen()),
+ input_dim_padded.getFeatureLen(), true);
+ if (status != ML_ERROR_NONE)
+ throw std::runtime_error("Backwarding Convolution failed.");
+ }
+
+ if (trainable) {
+ // Update K / bias
+ for (unsigned int i = 0; i < filter_size; ++i) {
+ Tensor &delK = weightAt(i).getGradientRef();
+ Tensor &filters = weightAt(i).getVariableRef();
+
+ if (isWeightRegularizerL2Norm()) {
+ status = delK.add_i(filters, weight_regularizer_constant);
+ if (status != ML_ERROR_NONE)
+ throw std::runtime_error("Weight regularization failed");
+ }
+ }
+
+ opt->apply_gradients(weight_list, num_weights, iteration);
+ }
+
+ return {MAKE_SHARED_TENSOR(std::move(strip_pad(ret, padding.data())))};
+}
+
+void Conv2DLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<Conv2DLayer> from = std::static_pointer_cast<Conv2DLayer>(l);
+ this->filter_size = from->filter_size;
+ for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
+ this->kernel_size[i] = from->kernel_size[i];
+ this->stride[i] = from->stride[i];
+ this->padding[i] = from->padding[i];
+ }
+}
+
+int Conv2DLayer::setSize(int *size, PropertyType type) {
+ int status = ML_ERROR_NONE;
+ switch (type) {
+ case PropertyType::kernel_size:
+ for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
+ kernel_size[i] = size[i];
+ }
+ break;
+ case PropertyType::stride:
+ for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
+ stride[i] = size[i];
+ }
+ break;
+ case PropertyType::padding:
+ for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
+ padding[i] = size[i];
+ }
+ break;
+ default:
+ ml_loge("Error: Unknown Layer Property type");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+ return status;
+}
+
+int Conv2DLayer::setFilter(int f) {
+ int status = ML_ERROR_NONE;
+ if (f <= 0) {
+ ml_loge("Error: number of filters must be greater than 0");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+ filter_size = f;
+ return status;
+}
+
+void Conv2DLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::filters: {
+ if (!value.empty()) {
+ status = setUint(filter_size, value);
+ throw_status(status);
+ }
+ } break;
+ case PropertyType::kernel_size:
+ if (!value.empty()) {
+ status = getValues(CONV2D_DIM, value, (int *)(kernel_size.data()));
+ throw_status(status);
+ if (kernel_size[0] == 0 || kernel_size[1] == 0) {
+ throw std::invalid_argument(
+ "[Conv2DLayer] kernel_size must be greater than 0");
+ }
+ }
+ break;
+ case PropertyType::stride:
+ if (!value.empty()) {
+ status = getValues(CONV2D_DIM, value, (int *)(stride.data()));
+ throw_status(status);
+ if (stride[0] == 0 || stride[1] == 0) {
+ throw std::invalid_argument(
+ "[Conv2DLayer] stride must be greater than 0");
+ }
+ }
+ break;
+ case PropertyType::padding:
+ if (!value.empty()) {
+ status = getValues(CONV2D_DIM, value, (int *)(padding.data()));
+ throw_status(status);
+ }
+ break;
+ case PropertyType::normalization:
+ if (!value.empty()) {
+ status = setBoolean(normalization, value);
+ throw_status(status);
+ }
+ break;
+ case PropertyType::standardization:
+ if (!value.empty()) {
+ status = setBoolean(standardization, value);
+ throw_status(status);
+ }
+ break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+int Conv2DLayer::conv2d(float *in, TensorDim indim, const float *kernel,
+ TensorDim kdim, float *out, unsigned int const *stride,
+ float bias) {
+
+ int status = ML_ERROR_NONE;
+ unsigned int channel = indim.channel();
+ unsigned int height = indim.height();
+ unsigned int width = indim.width();
+ unsigned int k_width = kdim.width();
+ unsigned int k_height = kdim.height();
+ unsigned int o_width = ((indim.width() - kdim.width()) / stride[0] + 1);
+
+ if (indim.channel() != kdim.channel()) {
+ ml_loge("Error: Input and Kenel Dimension is not match!");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ // Optimizer This routine : There are lots of dulicated calculations
+ unsigned int I = 0;
+ unsigned int J = 0;
+ for (unsigned int j = 0; j < height - k_height + 1; j += stride[0]) {
+ J = 0;
+ for (unsigned int k = 0; k < width - k_width + 1; k += stride[1]) {
+ float sum = 0.0f;
+ for (unsigned int i = 0; i < channel; ++i) {
+ for (unsigned int ki = 0; ki < k_height; ++ki) {
+ for (unsigned int kj = 0; kj < k_width; ++kj) {
+ sum += kernel[i * k_height * k_width + ki * k_width + kj] *
+ in[i * height * width + (j + ki) * width + (k + kj)];
+ }
+ }
+ }
+ sum += bias;
+ out[I * o_width + J] = sum;
+ J++;
+ }
+ I++;
+ }
+ return status;
+}
+
+int Conv2DLayer::conv2d_gemm(
+ const float *mkernel, TensorDim kdim, Tensor const &in, TensorDim outdim,
+ const std::array<unsigned int, CONV2D_DIM> &mstride,
+ const std::array<unsigned int, CONV2D_DIM> &pad, float *out,
+ unsigned int osize, bool channel_mode) {
+ int status = ML_ERROR_NONE;
+ std::vector<float> in_col;
+
+ if (channel_mode) {
+ in_col.resize(kdim.getFeatureLen() * outdim.width() * outdim.height());
+ } else {
+ in_col.resize(kdim.width() * kdim.height() * outdim.width());
+ }
+
+ Tensor in_padded = zero_pad(0, in, pad.data());
+ status =
+ im2col(in_padded, kdim, in_col.data(), outdim, mstride, channel_mode);
+ if (status != ML_ERROR_NONE)
+ throw std::runtime_error("Forwarding Convolution failed.");
+
+ float alpha_dgemm = 1.0f;
+ float beta_dgemm = 0.0f;
+ const float *data = mkernel;
+ const float *mdata = in_col.data();
+ float *rdata = out;
+
+ unsigned int kh, kw, w;
+
+ if (channel_mode) {
+ kh = kdim.batch();
+ kw = kdim.getFeatureLen();
+ w = outdim.width() * outdim.height();
+ } else {
+ kh = outdim.height();
+ kw = kdim.width() * kdim.height();
+ w = outdim.width();
+ }
+
+ sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, kh, w, kw, alpha_dgemm, data,
+ kw, mdata, w, beta_dgemm, rdata, w);
+
+ return status;
+}
+
+int Conv2DLayer::im2col(Tensor in_padded, TensorDim kdim, float *in_col,
+ TensorDim outdim,
+ const std::array<unsigned int, CONV2D_DIM> &mstride,
+ bool channel_mode) {
+
+ int status = ML_ERROR_NONE;
+ unsigned int count;
+ unsigned int channel = in_padded.channel();
+ unsigned int height = in_padded.height();
+ unsigned int width = in_padded.width();
+ unsigned int k_width = kdim.width();
+ unsigned int k_height = kdim.height();
+
+ unsigned int J = 0;
+ if (channel_mode) {
+ for (unsigned int j = 0; j <= height - k_height; j += mstride[0]) {
+ for (unsigned int k = 0; k <= width - k_width; k += mstride[1]) {
+ count = 0;
+ for (unsigned int i = 0; i < channel; ++i) {
+ for (unsigned int ki = 0; ki < k_height; ++ki) {
+ for (unsigned int kj = 0; kj < k_width; ++kj) {
+ in_col[count * (outdim.width() * outdim.height()) + J] =
+ in_padded
+ .getData()[i * height * width + (j + ki) * width + (k + kj)];
+ count++;
+ }
+ }
+ }
+ J++;
+ }
+ }
+ if (J != outdim.width() * outdim.height())
+ status = ML_ERROR_INVALID_PARAMETER;
+ } else {
+ for (unsigned int i = 0; i < channel; ++i) {
+ for (unsigned int j = 0; j <= height - k_height; j += mstride[0]) {
+ for (unsigned int k = 0; k <= width - k_width; k += mstride[1]) {
+ count = 0;
+ for (unsigned int ki = 0; ki < k_height; ++ki) {
+ for (unsigned int kj = 0; kj < k_width; ++kj) {
+ in_col[count * (outdim.width()) + J] =
+ in_padded
+ .getData()[i * height * width + (j + ki) * width + (k + kj)];
+ count++;
+ }
+ }
+ J++;
+ }
+ }
+ }
+ if (J != outdim.width())
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return status;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file conv2d_layer.h
+ * @date 01 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Convolution Layer Class for Neural Network
+ *
+ */
+
+#ifndef __CONV2D_LAYER_H_
+#define __CONV2D_LAYER_H_
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+constexpr const unsigned int CONV2D_DIM = 2;
+
+/**
+ * @class Convolution 2D Layer
+ * @brief Convolution 2D Layer
+ */
+class Conv2DLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Conv 2D Layer
+ */
+ template <typename... Args>
+ Conv2DLayer(unsigned int filter_size_ = 0,
+ const std::array<unsigned int, CONV2D_DIM> &kernel_size_ = {0, 0},
+ const std::array<unsigned int, CONV2D_DIM> &stride_ = {1, 1},
+ const std::array<unsigned int, CONV2D_DIM> &padding_ = {0, 0},
+ bool normalization_ = false, bool standardization_ = false,
+ Args... args) :
+ Layer(LayerType::LAYER_CONV2D, args...),
+ filter_size(filter_size_),
+ kernel_size(kernel_size_),
+ stride(stride_),
+ padding(padding_),
+ normalization(normalization_),
+ standardization(standardization_) {}
+
+ /**
+ * @brief Destructor of Conv 2D Layer
+ */
+ ~Conv2DLayer() {}
+
+ /**
+ * @brief Move constructor of Conv 2D Layer.
+ * @param[in] Conv2dLayer &&
+ */
+ Conv2DLayer(Conv2DLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs Conv2DLayer to be moved.
+ */
+ Conv2DLayer &operator=(Conv2DLayer &&rhs) = default;
+
+ /**
+ * @brief initialize layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file);
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file);
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /* TO DO : support keras type of padding */
+ /* enum class PaddingType { */
+ /* full = 0, */
+ /* same = 1, */
+ /* valid = 2, */
+ /* unknown = 3, */
+ /* }; */
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Convolution2D"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ unsigned int filter_size;
+ std::array<unsigned int, CONV2D_DIM> kernel_size;
+ std::array<unsigned int, CONV2D_DIM> stride;
+ std::array<unsigned int, CONV2D_DIM> padding;
+
+ bool normalization;
+ bool standardization;
+
+ /**
+ * @brief set Parameter Size
+ * @param[in] * size : size arrary
+ * @param[in] type : Property type
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setSize(int *size, PropertyType type);
+
+ /**
+ * @brief set Parameter Size
+ * @param[in] f number of filters
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setFilter(int f);
+
+ /**
+ * @brief set normalization
+ * @param[in] enable boolean
+ */
+ void setNormalization(bool enable) { this->normalization = enable; };
+
+ /**
+ * @brief set standardization
+ * @param[in] enable boolean
+ */
+ void setStandardization(bool enable) { this->standardization = enable; };
+
+ /**
+ * @brief calculation convolution
+ * @param[in] in input tensor data
+ * @param[in] indim input tensor dimension
+ * @param[in] kernel convolution kernel data
+ * @param[in] kdim convolution kernel dimension
+ * @param[in] out output
+ * @param[in] stride stride value : x, y direction
+ * @param[in] bias bias data
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int conv2d(float *in, TensorDim indim, const float *kernel, TensorDim kdim,
+ float *out, unsigned int const *stride, float bias);
+
+ /**
+ * @brief calculation convolution with cblas_*gemm
+ * @param[in] mkernel kernel data
+ * @param[in] kdim kernel data demension
+ * @param[in] in input tensor
+ * @param[in] outdim output tensor dimension
+ * @param[in] stride stride value : x, y direction
+ * @param[in] padd pad value : x, y direction
+ * @param[out] out output data
+ * @param[in] osize output size
+ * @param[in] channel_mode loop with channel first
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int conv2d_gemm(const float *mkernel, TensorDim kdim, Tensor const &in,
+ TensorDim outdim,
+ const std::array<unsigned int, CONV2D_DIM> &stride,
+ const std::array<unsigned int, CONV2D_DIM> &pad, float *out,
+ unsigned int osize, bool channel_mode);
+
+ /**
+ * @brief reform the data to 2d matrix
+ * @param[in] in_padded padded input data
+ * @param[in] kdim kernel dimesion for define number of row
+ * @param[out] inCol reformed data
+ * @param[in] outdim output dimension
+ * @param[in] mstride stride value : x, y direction
+ * @param[in] channel_mode loop with channel first
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int im2col(Tensor in_padded, TensorDim kdim, float *inCol, TensorDim outdim,
+ const std::array<unsigned int, CONV2D_DIM> &mstride,
+ bool channel_mode);
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __CONV2D_LAYER_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file fc_layer.cpp
+ * @date 14 May 2020
+ * @brief This is Fully Connected Layer Class for Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <fc_layer.h>
+#include <layer_internal.h>
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+enum class FCParams { weight, bias };
+
+int FullyConnectedLayer::initialize() {
+ int status = ML_ERROR_NONE;
+
+ if (num_inputs != 1) {
+ throw std::invalid_argument("Fully connected layer takes only one input");
+ }
+
+ output_dim[0] = input_dim[0];
+ output_dim[0].width(unit);
+
+ TensorDim bias_dim = TensorDim();
+ bias_dim.setTensorDim(3, unit);
+
+ TensorDim dim = output_dim[0];
+ dim.height(input_dim[0].width());
+ dim.batch(1);
+
+ setNumWeights(2);
+ weightAt(0) = std::move(Weight(dim, weight_initializer, true, "FC:weight"));
+ weightAt(1) = std::move(Weight(bias_dim, bias_initializer, true, "FC:bias"));
+
+ return status;
+}
+
+void FullyConnectedLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+ switch (type) {
+ case PropertyType::unit: {
+ if (!value.empty()) {
+ status = setUint(unit, value);
+ throw_status(status);
+ output_dim[0].width(unit);
+ }
+ } break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+sharedConstTensors FullyConnectedLayer::forwarding(sharedConstTensors in) {
+ Tensor &weight =
+ weightAt(static_cast<int>(FCParams::weight)).getVariableRef();
+ Tensor &bias = weightAt(static_cast<int>(FCParams::bias)).getVariableRef();
+
+ input = *in[0];
+ hidden = input.dot(weight);
+ hidden.add_i(bias);
+
+ if (weight_regularizer == WeightRegularizerType::l2norm) {
+ loss = weight_regularizer_constant * 0.5f * (weight.l2norm());
+ }
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+void FullyConnectedLayer::read(std::ifstream &file) {
+ Layer::read(file);
+ if (opt)
+ opt->read(file);
+}
+
+void FullyConnectedLayer::save(std::ofstream &file) {
+ Layer::save(file);
+ if (opt)
+ opt->save(file);
+}
+
+void FullyConnectedLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<FullyConnectedLayer> from =
+ std::static_pointer_cast<FullyConnectedLayer>(l);
+ this->unit = from->unit;
+}
+
+sharedConstTensors
+FullyConnectedLayer::backwarding(sharedConstTensors derivative, int iteration) {
+ unsigned int weight_idx = static_cast<int>(FCParams::weight);
+ unsigned int bias_idx = static_cast<int>(FCParams::bias);
+ Tensor &weight = weightAt(weight_idx).getVariableRef();
+ Tensor &djdw = weightAt(weight_idx).getGradientRef();
+ Tensor &djdb = weightAt(bias_idx).getGradientRef();
+
+ Tensor ret = derivative[0]->dot(weight, false, true);
+ djdb = derivative[0]->sum(0);
+
+ djdw = input.dot(*derivative[0], true, false);
+ if (isWeightRegularizerL2Norm())
+ djdw.add_i(weight, weight_regularizer_constant);
+ djdw = djdw.sum(0);
+
+ if (trainable) {
+ opt->apply_gradients(weight_list, num_weights, iteration);
+ }
+
+ return {MAKE_SHARED_TENSOR(std::move(ret))};
+}
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file fc_layer.h
+ * @date 14 May 2020
+ * @brief This is Fully Connected Layer Class of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __FC_LAYER_H__
+#define __FC_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class FullyConnecedLayer
+ * @brief fully connected layer
+ */
+class FullyConnectedLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Fully Connected Layer
+ */
+ template <typename... Args>
+ FullyConnectedLayer(unsigned int unit_ = 0, Args... args) :
+ Layer(LayerType::LAYER_FC, args...),
+ unit(unit_) {}
+
+ /**
+ * @brief Destructor of Fully Connected Layer
+ */
+ ~FullyConnectedLayer(){};
+
+ /**
+ * @brief Move constructor.
+ * @param[in] FullyConnected &&
+ */
+ FullyConnectedLayer(FullyConnectedLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs FullyConnectedLayer to be moved.
+ */
+ FullyConnectedLayer &operator=(FullyConnectedLayer &&rhs) = default;
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file);
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file);
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /**
+ * @brief initialize layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "FullyConnected"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ unsigned int unit;
+};
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __FC_LAYER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file flatten_layer.cpp
+ * @date 16 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Flatten Layer Class for Neural Network
+ *
+ */
+
+#include <flatten_layer.h>
+#include <layer_internal.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int FlattenLayer::initialize() {
+ if (num_inputs != 1) {
+ throw std::invalid_argument("input_shape keyword is only for one input");
+ }
+
+ TensorDim &out_dim = output_dim[0];
+ int status = ML_ERROR_NONE;
+ if (input_dim[0].getDataLen() == 1) {
+ ml_logw("Warning: the length of previous layer dimension is one");
+ }
+
+ out_dim.batch(input_dim[0].batch());
+ out_dim.channel(1);
+ out_dim.height(1);
+ out_dim.width(input_dim[0].getFeatureLen());
+
+ return status;
+}
+
+sharedConstTensors FlattenLayer::forwarding(sharedConstTensors in) {
+ input = *in[0];
+ hidden = input;
+
+ hidden.reshape(output_dim[0]);
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors FlattenLayer::backwarding(sharedConstTensors in,
+ int iteration) {
+ Tensor temp = *in[0];
+ temp.reshape(input_dim[0]);
+
+ return {MAKE_SHARED_TENSOR(std::move(temp))};
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file flatten_layer.h
+ * @date 16 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Flatten Layer Class for Neural Network
+ *
+ */
+
+#ifndef __FLATTEN_LAYER_H__
+#define __FLATTEN_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class Flatten Layer
+ * @brief Flatten Layer
+ */
+class FlattenLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Flatten Layer
+ */
+ template <typename... Args>
+ FlattenLayer(Args... args) : Layer(LayerType::LAYER_FLATTEN, args...) {}
+
+ /**
+ * @brief Destructor of Flatten Layer
+ */
+ ~FlattenLayer(){};
+
+ /**
+ * @brief Move constructor of FlattenLayer.
+ * @param[in] FlattenLayer &&
+ */
+ FlattenLayer(FlattenLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs FlattenLayer to be moved.
+ */
+ FlattenLayer &operator=(FlattenLayer &&rhs) = default;
+
+ /**
+ * @brief initialize layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file){};
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file){};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Flatten"; };
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __FLATTEN_LAYER_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file input_layer.cpp
+ * @date 14 May 2020
+ * @brief This is Input Layer Class for Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <input_layer.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+
+namespace nntrainer {
+
+void InputLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::normalization:
+ if (!value.empty()) {
+ status = setBoolean(normalization, value);
+ throw_status(status);
+ }
+ break;
+ case PropertyType::standardization:
+ if (!value.empty()) {
+ status = setBoolean(standardization, value);
+ throw_status(status);
+ }
+ break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+sharedConstTensors InputLayer::forwarding(sharedConstTensors in) {
+ input = *in[0];
+
+ hidden = input;
+ if (normalization)
+ hidden = hidden.normalization();
+ if (standardization)
+ hidden = hidden.standardization();
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors InputLayer::backwarding(sharedConstTensors in,
+ int iteration) {
+ return in;
+}
+
+int InputLayer::initialize() {
+ int status = ML_ERROR_NONE;
+ output_dim = input_dim;
+
+ return status;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file input_layer.h
+ * @date 14 May 2020
+ * @brief This is Input Layer Class of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __INPUT_LAYER_H__
+#define __INPUT_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class Input Layer
+ * @brief Just Handle the Input of Network
+ */
+class InputLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of InputLayer
+ */
+ template <typename... Args>
+ InputLayer(bool normalization = false, bool standardization = false,
+ Args... args) :
+ Layer(LayerType::LAYER_IN, args...),
+ normalization(false),
+ standardization(false) {}
+
+ /**
+ * @brief Destructor of InputLayer
+ */
+ ~InputLayer() {}
+
+ /**
+ * @brief Move constructor of Pooling 2D Layer.
+ * @param[in] Input &&
+ */
+ InputLayer(InputLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs InputLayer to be moved.
+ */
+ InputLayer &operator=(InputLayer &&rhs) = default;
+
+ /**
+ * @brief No Weight data for this Input Layer
+ */
+ void read(std::ifstream &file){};
+
+ /**
+ * @brief No Weight data for this Input Layer
+ */
+ void save(std::ofstream &file){};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief Initializer of Input Layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Input"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ bool normalization;
+ bool standardization;
+
+ /**
+ * @brief set normalization
+ * @param[in] enable boolean
+ */
+ void setNormalization(bool enable) { this->normalization = enable; };
+
+ /**
+ * @brief set standardization
+ * @param[in] enable boolean
+ */
+ void setStandardization(bool enable) { this->standardization = enable; };
+};
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __INPUT_LAYER_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file layer.cpp
+ * @date 04 December 2019
+ * @brief This is Layers Classes for Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <layer_internal.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <optimizer_factory.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+void Layer::setActivation(ActivationType acti) {
+ if (acti == ActivationType::ACT_UNKNOWN) {
+ throw std::invalid_argument("Error:have to specify activation function");
+ }
+ activation_type = acti;
+}
+
+int Layer::setOptimizer(std::shared_ptr<Optimizer> opt) {
+ this->opt = createOptimizer(opt->getType(), *opt.get());
+ return this->opt->initialize(weight_list, num_weights, true);
+}
+
+int Layer::checkValidation() {
+ int status = ML_ERROR_NONE;
+ if (type == LayerType::LAYER_UNKNOWN) {
+ ml_loge("Error: Layer type is unknown");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ if (activation_type == ActivationType::ACT_UNKNOWN) {
+ ml_loge("Error: Have to set activation for this layer");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return status;
+}
+
+void Layer::setBatch(unsigned int batch) {
+ for (unsigned int idx = 0; idx < num_inputs; ++idx)
+ input_dim[idx].setTensorDim(0, batch);
+
+ for (unsigned int idx = 0; idx < num_outputs; ++idx)
+ output_dim[idx].setTensorDim(0, batch);
+}
+
+void Layer::copy(std::shared_ptr<Layer> l) {
+ setNumWeights(l->num_weights);
+ for (unsigned int i = 0; i < num_weights; ++i) {
+ weightAt(i) = l->weightAt(i);
+ }
+
+ // TODO: fix this #630
+ this->opt = l->opt;
+ this->input_dim = l->input_dim;
+ this->output_dim = l->output_dim;
+ this->input.copy(l->input);
+ this->hidden.copy(l->hidden);
+ this->activation_type = l->activation_type;
+ this->loss = l->loss;
+ this->type = l->type;
+ this->weight_regularizer = l->weight_regularizer;
+ this->weight_regularizer_constant = l->weight_regularizer_constant;
+ this->weight_initializer = l->weight_initializer;
+ this->flatten = l->flatten;
+ this->trainable = l->trainable;
+ this->num_inputs = l->num_inputs;
+ this->num_outputs = l->num_outputs;
+}
+
+void Layer::read(std::ifstream &file) {
+ for (unsigned int i = 0; i < num_weights; ++i) {
+ weightAt(i).getVariableRef().read(file);
+ }
+}
+
+void Layer::save(std::ofstream &file) {
+ for (unsigned int i = 0; i < num_weights; ++i) {
+ weightAt(i).getVariableRef().save(file);
+ }
+}
+
+int Layer::setProperty(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+
+ status = getKeyValue(values[i], key, value);
+ NN_RETURN_STATUS();
+
+ unsigned int type = parseLayerProperty(key);
+
+ if (value.empty()) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ try {
+ /// @note this calls derived setProperty if available
+ setProperty(static_cast<PropertyType>(type), value);
+ } catch (...) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ }
+ return status;
+}
+
+void Layer::setProperty(const PropertyType type, const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::name:
+ if (!value.empty()) {
+ status = setName(value);
+ throw_status(status);
+ }
+ break;
+ case PropertyType::input_shape: {
+ if (num_inputs != 1) {
+ throw std::invalid_argument("input_shape keyword is only for one input");
+ }
+
+ TensorDim &in_dim = input_dim[0];
+ if (!value.empty()) {
+ unsigned int cache_batch_size = 1;
+ /** cache original value of batch size */
+ if (in_dim.batch()) {
+ cache_batch_size = in_dim.batch();
+ in_dim.batch(1);
+ }
+ status = in_dim.setTensorDim(value.c_str());
+ if (in_dim.batch() > 1) {
+ ml_logw("Batch size set with input dimension %d is ignored."
+ "Set batchsize property for the model to update batchsize.",
+ in_dim.batch());
+ }
+ /** set back to cache value of dimension */
+ in_dim.batch(cache_batch_size);
+ throw_status(status);
+ }
+ } break;
+ case PropertyType::activation:
+ if (!value.empty()) {
+ setActivation((ActivationType)parseType(value, TOKEN_ACTI));
+ }
+ break;
+ case PropertyType::flatten:
+ if (!value.empty()) {
+ status = setBoolean(flatten, value);
+ throw_status(status);
+ }
+ break;
+ case PropertyType::weight_regularizer:
+ if (!value.empty()) {
+ weight_regularizer =
+ (WeightRegularizerType)parseType(value, TOKEN_WEIGHT_REGULARIZER);
+ if (weight_regularizer == WeightRegularizerType::unknown) {
+ throw std::invalid_argument("[Layer] Unknown Weight decay");
+ }
+ }
+ break;
+ case PropertyType::weight_regularizer_constant:
+ if (!value.empty()) {
+ status = setFloat(weight_regularizer_constant, value);
+ throw_status(status);
+ }
+ break;
+ case PropertyType::weight_initializer:
+ if (!value.empty()) {
+ weight_initializer =
+ (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
+ }
+ break;
+ case PropertyType::bias_initializer:
+ if (!value.empty()) {
+ bias_initializer = (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
+ }
+ break;
+ case PropertyType::input_layers:
+ if (!value.empty()) {
+ std::regex reg("\\,+");
+ std::vector<std::string> concat_layers = split(value, reg);
+ // TODO set num_inputs properly
+ num_inputs = 1;
+ if (concat_layers.size() > 1)
+ num_inputs = concat_layers.size();
+ }
+ break;
+ default:
+ std::string msg =
+ "[Layer] Unknown Layer Property Key for value " + std::string(value);
+ throw exception::not_supported(msg);
+ }
+}
+
+int Layer::setName(std::string name_) {
+ if (name_.empty())
+ return ML_ERROR_INVALID_PARAMETER;
+
+ name = name_;
+ return ML_ERROR_NONE;
+}
+
+template <typename T>
+void Layer::printIfValid(std::ostream &out, const PropertyType type,
+ const T target) {
+ try {
+ setProperty(type);
+ } catch (exception::not_supported &e) {
+ return;
+ }
+
+ out << propToStr(static_cast<unsigned int>(type)) << ": " << target
+ << std::endl;
+}
+
+void Layer::printShapeInfo(std::ostream &out) {
+ for (unsigned int idx = 0; idx < num_inputs; ++idx) {
+ out << "input " << input_dim[idx];
+ for (unsigned int i = 0; i < num_weights; i++)
+ out << "inner" << i << " " << weightAt(i).var.getDim();
+ }
+ for (unsigned int idx = 0; idx < num_outputs; ++idx) {
+ out << "output " << output_dim[idx];
+ }
+}
+
+void Layer::printPropertiesMeta(std::ostream &out) {
+ printIfValid(
+ out, PropertyType::activation,
+ static_cast<std::underlying_type<ActivationType>::type>(activation_type));
+ printIfValid(out, PropertyType::flatten, flatten);
+}
+
+void Layer::printProperties(std::ostream &out) {
+ out << "Trainable: " << trainable << std::endl;
+ printIfValid(out, PropertyType::weight_regularizer,
+ static_cast<int>(weight_regularizer));
+ printIfValid(out, PropertyType::weight_regularizer_constant,
+ weight_regularizer_constant);
+}
+
+void Layer::printMetric(std::ostream &out) {
+ if (loss > 0) {
+ out << "Weight regularization loss: " << loss;
+ }
+}
+
+void Layer::printPreset(std::ostream &out, PrintPreset preset) {
+ unsigned int flags = 0;
+ switch (preset) {
+ case PrintPreset::PRINT_ALL:
+ flags = PRINT_WEIGHTS | PRINT_METRIC;
+ /// fall through intended
+ case PrintPreset::PRINT_SUMMARY_META:
+ flags |= PRINT_PROP_META;
+ /// fall through intended
+ case PrintPreset::PRINT_SUMMARY:
+ flags |= PRINT_INST_INFO | PRINT_SHAPE_INFO | PRINT_PROP | PRINT_PROP_META;
+ break;
+ case PrintPreset::PRINT_NONE:
+ return;
+ default:
+ throw ::std::invalid_argument("undefined preset given");
+ }
+ print(out, flags);
+}
+
+void Layer::print(std::ostream &out, unsigned int flags) {
+ if (flags & PRINT_INST_INFO) {
+ out << "===================";
+ if (getName().empty())
+ printInstance(out, this);
+ else
+ out << "<" << getName() << ">" << std::endl;
+
+ out << "Layer Type: "
+ << static_cast<std::underlying_type<LayerType>::type>(type)
+ << std::endl;
+ }
+
+ if (flags & PRINT_SHAPE_INFO) {
+ out << "======shape information: " << std::endl;
+ printShapeInfo(out);
+ }
+
+ if (flags & PRINT_PROP_META) {
+ out << "======meta properties: " << std::endl;
+ printPropertiesMeta(out);
+ }
+
+ if (flags & PRINT_PROP) {
+ out << "======properties: " << std::endl;
+ printProperties(out);
+ }
+
+ if (flags & PRINT_WEIGHTS) {
+ out << "======weights: " << std::endl;
+ for (unsigned int i = 0; i < num_weights; ++i) {
+ out << '[' << weightAt(i).getName() << ']' << std::endl;
+ out << weightAt(i).var;
+ }
+ }
+
+ if (flags & PRINT_METRIC) {
+ out << "======metrics: " << std::endl;
+ printMetric(out);
+ }
+};
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file optimizer_factory.cpp
+ * @date 11 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the layer factory.
+ */
+
+#include <layer_factory.h>
+
+#include <activation_layer.h>
+#include <addition_layer.h>
+#include <bn_layer.h>
+#include <conv2d_layer.h>
+#include <fc_layer.h>
+#include <flatten_layer.h>
+#include <input_layer.h>
+#include <loss_layer.h>
+#include <nntrainer_error.h>
+#include <pooling2d_layer.h>
+
+#ifdef ENABLE_TFLITE_BACKBONE
+#include <tflite_layer.h>
+#endif
+
+#ifdef ENABLE_NNSTREAMER_BACKBONE
+#include <nnstreamer_layer.h>
+#endif
+
+namespace nntrainer {
+
+/**
+ * @brief Factory creator with constructor
+ */
+std::unique_ptr<Layer> createLayer(LayerType type) {
+ switch (type) {
+ case LayerType::LAYER_IN:
+ return std::make_unique<InputLayer>();
+ case LayerType::LAYER_FC:
+ return std::make_unique<FullyConnectedLayer>();
+ case LayerType::LAYER_BN:
+ return std::make_unique<BatchNormalizationLayer>();
+ case LayerType::LAYER_CONV2D:
+ return std::make_unique<Conv2DLayer>();
+ case LayerType::LAYER_POOLING2D:
+ return std::make_unique<Pooling2DLayer>();
+ case LayerType::LAYER_FLATTEN:
+ return std::make_unique<FlattenLayer>();
+ case LayerType::LAYER_ACTIVATION:
+ return std::make_unique<ActivationLayer>();
+ case LayerType::LAYER_ADDITION:
+ return std::make_unique<AdditionLayer>();
+ case LayerType::LAYER_LOSS:
+ return std::make_unique<LossLayer>();
+ case LayerType::LAYER_BACKBONE_NNSTREAMER:
+#ifdef ENABLE_NNSTREAMER_BACKBONE
+ return std::make_unique<NNStreamerLayer>();
+#else
+ throw exception::not_supported("NNStreamer backbone layer not supported");
+#endif
+ case LayerType::LAYER_BACKBONE_TFLITE:
+#ifdef ENABLE_TFLITE_BACKBONE
+ return std::make_unique<TfLiteLayer>();
+#else
+ throw exception::not_supported("TfLite backbone layer not supported");
+#endif
+ case LayerType::LAYER_UNKNOWN:
+ /** fallthrough intended */
+ default:
+ throw std::invalid_argument("Unknown type for the layer");
+ }
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file layer_factory.h
+ * @date 7 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the layer factory.
+ */
+
+#ifndef __LAYER_FACTORY_H__
+#define __LAYER_FACTORY_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Factory creator with constructor
+ */
+std::unique_ptr<Layer> createLayer(LayerType type);
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __LAYER_FACTORY_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file layer_internal.h
+ * @date 04 December 2019
+ * @brief This is Layer classes of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+#ifndef __LAYER_H__
+#define __LAYER_H__
+#ifdef __cplusplus
+
+#include <memory>
+#include <set>
+#include <vector>
+
+#include <layer.h>
+#include <optimizer_internal.h>
+#include <tensor.h>
+#include <tensor_dim.h>
+#include <weight.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Enumeration of activation function type
+ */
+enum class ActivationType {
+ ACT_TANH, /** tanh */
+ ACT_SIGMOID, /** sigmoid */
+ ACT_RELU, /** ReLU */
+ ACT_SOFTMAX, /** softmax */
+ ACT_NONE, /** no op */
+ ACT_UNKNOWN /** unknown */
+};
+
+using LayerType = ml::train::LayerType;
+
+/**
+ * @class Layer Base class for layers
+ * @brief Base class for all layers
+ */
+class Layer : public ml::train::Layer {
+
+ /** model classes can call private methods which arent exposed to public */
+ friend class NeuralNetwork;
+ friend class ModelLoader;
+
+public:
+ /**
+ * @brief Constructor of Layer Class
+ */
+ Layer(
+ LayerType type_, ActivationType activation_type_ = ActivationType::ACT_NONE,
+ WeightRegularizerType weight_regularizer_ = WeightRegularizerType::unknown,
+ const float weight_regularizer_constant_ = 1.0f,
+ WeightInitializer weight_initializer_ =
+ WeightInitializer::WEIGHT_XAVIER_UNIFORM,
+ WeightInitializer bias_initializer_ = WeightInitializer::WEIGHT_ZEROS,
+ bool trainable_ = true, bool flatten_ = false) :
+ name(std::string()),
+ type(type_),
+ loss(0.0f),
+ activation_type(activation_type_),
+ weight_regularizer(weight_regularizer_),
+ weight_regularizer_constant(weight_regularizer_constant_),
+ weight_initializer(weight_initializer_),
+ bias_initializer(bias_initializer_),
+ flatten(flatten_),
+ trainable(trainable_),
+ num_weights(0),
+ num_inputs(1),
+ num_outputs(1) {
+ input_dim.resize(1);
+ output_dim.resize(1);
+ }
+
+ /**
+ * @brief Move constructor of Layer.
+ * @param[in] Layer &&
+ */
+ Layer(Layer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs Layer to be moved.
+ */
+ virtual Layer &operator=(Layer &&rhs) = default;
+
+ /**
+ * @brief Forward Propagation of a layer
+ * @param[in] in List of Input Tensors taken by this layer
+ * @retval List of Output Tensors
+ */
+ virtual sharedConstTensors forwarding(sharedConstTensors in) = 0;
+
+ /**
+ * @brief Back Propagation of a layer
+ * @param[in] in List of Derivative Tensor from the next layer
+ * @param[in] iteration Iteration value for the Optimizer
+ * @retval Derivative List of Tensor for the previous layer
+ */
+ virtual sharedConstTensors backwarding(sharedConstTensors in,
+ int iteration) = 0;
+
+ /**
+ * @brief read layer Weight & Bias data from file
+ * @note derived class can call this to get/save weights
+ * @param[in] file input file stream
+ */
+ virtual void read(std::ifstream &file);
+
+ /**
+ * @brief save layer Weight & Bias data from file
+ * @note derived class can call this to get/save weights
+ * @param[in] file output file stream
+ */
+ virtual void save(std::ofstream &file);
+
+ /**
+ * @brief set Property of layer
+ * @param[in] values values of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(std::vector<std::string> values);
+
+ /**
+ * @brief setProperty by PropertyType
+ * @note By passing empty string, this can validate if @a type is valid
+ * @param[in] type property type to be passed
+ * @param[in] value value to be passed, if empty string is passed, do nothing
+ * but throws error when @a type is invalid
+ * @exception exception::not_supported when property type is not valid for
+ * the particular layer
+ * @exception std::invalid_argument invalid argument
+ */
+ virtual void setProperty(const PropertyType type,
+ const std::string &value = "");
+
+ /**
+ * @brief Optimizer Setter
+ * @param[in] opt Optimizer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setOptimizer(std::shared_ptr<Optimizer> opt);
+
+ /**
+ * @brief Get the Optimizer object
+ *
+ * @return std::shared_ptr<Optimizer> optimizer
+ */
+ std::shared_ptr<Optimizer> getOptimizer() { return opt; }
+
+ /**
+ * @brief Activation Type Getter
+ * @retval Activation Type.
+ */
+ ActivationType getActivationType() { return this->activation_type; }
+
+ /**
+ * @brief Layer type Getter
+ * @retval type LayerType
+ */
+ LayerType getType() { return type; }
+
+ /**
+ * @brief Copy Layer
+ * @param[in] l Layer to be copied
+ */
+ virtual void copy(std::shared_ptr<Layer> l);
+
+ /**
+ * @brief check hyper parameter for the layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int checkValidation();
+
+ /**
+ * @brief Get the output dimension
+ * @return TensorDim dimension of the output
+ */
+ std::vector<TensorDim> getOutputDimension() { return output_dim; }
+
+ /**
+ * @brief Get the input dimension
+ * @return TensorDim dimension of the input
+ */
+ std::vector<TensorDim> getInputDimension() { return input_dim; }
+
+ /**
+ * @brief get the loss value added by this layer
+ * @retval loss value
+ */
+ float getLoss() { return loss; }
+
+ /**
+ * @brief set trainable for this layer
+ * @param[in] train to enable/disable train
+ */
+ virtual void setTrainable(bool train) { trainable = train; }
+
+ /**
+ * @brief get trainable for this layer
+ * @retval train to enable/disable train
+ */
+ bool getTrainable() noexcept { return trainable; }
+
+ /**
+ * @brief get all weights of the layer
+ * @retval vector of all params
+ */
+ std::shared_ptr<Weight> getWeights() { return weight_list; }
+
+ /**
+ * @brief get if the output of this layer must be flatten
+ * @retval flatten value
+ */
+ bool getFlatten() { return flatten; }
+
+ /**
+ * @brief Set name of the layer
+ */
+ int setName(std::string name);
+
+ /**
+ * @brief Get name of the layer
+ */
+ std::string getName() noexcept { return name; }
+
+ /**
+ * @brief print using PrintPreset
+ *
+ * @param out oustream
+ * @param preset preset to be used
+ */
+ void printPreset(std::ostream &out,
+ PrintPreset preset = PrintPreset::PRINT_SUMMARY);
+
+ /**
+ * @brief get data alias at param position.
+ * @exception std::out_of_range for index out of range
+ */
+ Weight &weightAt(const unsigned int position) {
+ if (position >= num_weights) {
+ throw std::out_of_range("index out of range");
+ }
+
+ return weight_list.get()[position];
+ }
+
+ /**
+ * @brief Get the number of weights
+ *
+ * @return unsigned int number of weights
+ */
+ unsigned int getNumWeights() { return num_weights; }
+
+ /**
+ * @brief Set the batch for the layer
+ * @param batch Batch value to be set
+ * @todo Make this private. Only model should be able to do this.
+ */
+ virtual void setBatch(unsigned int batch);
+
+protected:
+ /**
+ * @brief Print Options when printing layer info
+ */
+ typedef enum {
+ // clang-format off
+ PRINT_INST_INFO = (1 << 0), /**< Option to print type & instance address info */
+ PRINT_SHAPE_INFO = (1 << 1), /**< Option to print shape information, invalid before initiation*/
+ PRINT_PROP = (1 << 2), /**< Option to print properties */
+ PRINT_PROP_META = (1 << 3), /**< Option to print properties that describe meta info
+ e.g) layer activation type for non-activation layer. */
+ PRINT_WEIGHTS = (1 << 4), /**< Option to print weights */
+ PRINT_METRIC = (1 << 5) /**< Option to print metrics (currently loss only) */
+ // clang-format on
+ } PrintOption;
+
+ /**
+ * @brief Name of the layer (works as the identifier)
+ */
+ std::string name;
+
+ /**
+ * @brief check if current layer's weight decay type is l2norm
+ * @return bool is weightdecay type is L2 Norm
+ */
+ bool isWeightRegularizerL2Norm() {
+ return weight_regularizer == WeightRegularizerType::l2norm;
+ }
+ /**
+ * @brief Input Tensor
+ */
+ Tensor input;
+
+ /**
+ * @brief Hidden Layer Tensor which store the
+ * forwading result
+ */
+ Tensor hidden;
+
+ /**
+ * @brief Dimension of input activation
+ */
+ std::vector<TensorDim> input_dim;
+
+ /**
+ * @brief Dimension of output activation
+ */
+ std::vector<TensorDim> output_dim;
+
+ /**
+ * @brief Optimizer for this layer
+ */
+ // TODO: fix with #630
+ std::shared_ptr<Optimizer> opt;
+
+ /**
+ * @brief Layer type
+ */
+ LayerType type;
+
+ /**
+ * @brief Loss value added by this layer
+ */
+ float loss;
+
+ ActivationType activation_type;
+
+ WeightRegularizerType weight_regularizer;
+
+ float weight_regularizer_constant;
+
+ WeightInitializer weight_initializer; /** initializer for weights */
+
+ WeightInitializer bias_initializer; /** initializer for bias */
+
+ /**
+ * @brief Output of this layer should be flattened
+ */
+ bool flatten;
+
+ /*
+ * @brief making this false will skip updating this layer variables
+ */
+ bool trainable;
+
+ /**
+ * @brief reserve memory for @a weight_list and set @a num_weights
+ * @exception std::invalid_argument when num_weights is already set and
+ * shouldn't be changed again.
+ */
+ void setNumWeights(unsigned int psize) {
+ if (psize == num_weights)
+ return;
+
+ if (num_weights > 0) {
+ throw std::invalid_argument("param size can't be set once it is set");
+ }
+
+ num_weights = psize;
+ weight_list = std::shared_ptr<Weight>(new Weight[num_weights],
+ std::default_delete<Weight[]>());
+ }
+
+ /**
+ * @brief weight_list in this layer. This contains trainable weights of
+ * layers.
+ */
+ std::shared_ptr<Weight> weight_list;
+
+ unsigned int num_weights; /**< length of weights.
+ This shouldn't be changed
+ after initiation
+ use setNumWeights() to avoid
+ setting parameters twice */
+
+ /**
+ * @brief Number of inputs this layer will requries/will operate on
+ */
+ unsigned int num_inputs;
+
+ /**
+ * @brief Numer of outputs this layer will produce
+ */
+ unsigned int num_outputs;
+
+ /**
+ * @brief Layer type Setter
+ * @param[in] type layer type
+ */
+ void setType(LayerType type) { this->type = type; }
+
+ /**
+ * @brief Activation Setter
+ * @param[in] activation activation type
+ * @throw std::invalid_argument when ActivationType is unknown
+ */
+ virtual void setActivation(ActivationType activation);
+
+private:
+ /**
+ * @brief Set containing all the names of layers
+ */
+ static std::set<std::string> layer_names;
+
+ /**
+ * @brief Count assigned to layer names declared by default
+ */
+ static int def_name_count;
+
+ /**
+ * @brief Ensure that layer has a name
+ */
+ void ensureName();
+
+ /**
+ * @brief check if @a type is valid and print if prop is valid to @a out
+ */
+ template <typename T>
+ void printIfValid(std::ostream &out, const PropertyType type, T target);
+
+ /**
+ * @brief anchor point to override if PRINT_SHAPE_INFO is enabled for
+ * Layer::print()
+ */
+ virtual void printShapeInfo(std::ostream &out);
+
+ /**
+ * @brief anchor point to override if PRINT_PROP_META is enabled for
+ * Layer::print()
+ */
+ virtual void printPropertiesMeta(std::ostream &out);
+
+ /**
+ * @brief anchor point to override if PRINT_PROP is enabled for Layer::print()
+ */
+ virtual void printProperties(std::ostream &out);
+
+ /**
+ * @brief anchor point to override if PRINT_METRIC is enabled for
+ * Layer::print()
+ */
+ virtual void printMetric(std::ostream &out);
+
+ /**
+ * @brief set weight decay parameters
+ * @param[in] w struct for weight decay
+ */
+ void setWeightRegularizer(WeightRegularizerType type) {
+ weight_regularizer = type;
+ }
+
+ /**
+ * @brief set Weight Initialization Type
+ * @param[in] wini WeightInitializer
+ */
+ void setWeightInit(WeightInitializer wini) { weight_initializer = wini; }
+
+ /**
+ * @brief get if the output of this layer must be flatten
+ * @retval flatten value
+ */
+ void setFlatten(bool flatten) { this->flatten = flatten; }
+
+ /**
+ * @brief Print layer related information. Do not override without clear
+ * reason. It is recommended to override printShapeInfo, printPropertiesMeta,
+ * printProperties, printMetric instead
+ * @param[in] out outstream
+ * @param[in] flags combination of LayerPrintOption
+ */
+ virtual void print(std::ostream &out, unsigned int flags = 0);
+
+ /**
+ * @brief Get base name of the layer
+ */
+ virtual std::string getBaseName() = 0;
+
+ /**
+ * @brief Initialize the layer
+ * - Weight(Height, Width), Bias(1, Width)
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int initialize() = 0;
+
+ /**
+ * @brief Set the input dimension
+ * @param[in] d dimension to be set
+ */
+ void setInputDimension(std::vector<TensorDim> d) { input_dim = d; }
+};
+
+/**
+ * @brief Overriding output stream for layers and it's derived class
+ */
+template <typename T, typename std::enable_if_t<
+ std::is_base_of<Layer, T>::value, T> * = nullptr>
+std::ostream &operator<<(std::ostream &out, T &l) {
+ l.printPreset(out, Layer::PrintPreset::PRINT_SUMMARY);
+ return out;
+}
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __LAYER_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file loss_layer.cpp
+ * @date 12 June 2020
+ * @brief This is Loss Layer Class for Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <activation_layer.h>
+#include <cmath>
+#include <layer_internal.h>
+#include <lazy_tensor.h>
+#include <loss_layer.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int LossLayer::initialize() {
+ int status = ML_ERROR_NONE;
+
+ output_dim = input_dim;
+ return status;
+}
+
+sharedConstTensors LossLayer::forwarding(sharedConstTensors in,
+ sharedConstTensors label) {
+ input = *in[0];
+ Tensor y2 = *label[0];
+ Tensor y = input;
+ Tensor l;
+
+ switch (loss_type) {
+ case LossType::LOSS_MSE: {
+ // y2 <- y2 - y;
+ Tensor residual = y2.subtract(y);
+ l = residual.chain().multiply_i(residual).average().run();
+ } break;
+ case LossType::LOSS_ENTROPY_SIGMOID: {
+ // @todo: change this to apply_i
+ // @note: the output should be logit before applying sigmoid
+ // log(1 + exp(-abs(y))) + max(y, 0)
+ Tensor mid_term = y.apply(static_cast<float (*)(float)>(&std::fabs))
+ .multiply(-1.0)
+ .apply(static_cast<float (*)(float)>(&std::exp))
+ .add(1.0)
+ .apply(logFloat);
+ mid_term = mid_term.add(y.apply(ActivationLayer::relu));
+
+ // y * y2
+ Tensor end_term = y2.chain().multiply_i(y).run();
+
+ // loss = log(1 + exp(-abs(y))) + max(y, 0) - (y * y2)
+ l = mid_term.subtract(end_term).average();
+ y = y.apply(ActivationLayer::sigmoid);
+ } break;
+ case LossType::LOSS_ENTROPY_SOFTMAX: {
+ y = y.apply(ActivationLayer::softmax);
+ l = y2.chain()
+ .multiply_i(y.apply(logFloat))
+ .run()
+ .sum_by_batch()
+ .multiply(-1);
+
+ } break;
+ case LossType::LOSS_ENTROPY: {
+ throw std::runtime_error(
+ "Error: Cross Entropy not supported without softmax or sigmoid.");
+ }
+ case LossType::LOSS_UNKNOWN:
+ /** intended */
+ default: { throw std::runtime_error("Error: Unknown loss_type."); }
+ }
+
+ updateLoss(l);
+ return {MAKE_SHARED_TENSOR(std::move(y))};
+}
+
+sharedConstTensors LossLayer::forwarding(sharedConstTensors in) {
+ Tensor ret;
+
+ switch (loss_type) {
+ case LossType::LOSS_MSE:
+ return in;
+ case LossType::LOSS_ENTROPY_SIGMOID:
+ ret = in[0]->apply(ActivationLayer::sigmoid);
+ return {MAKE_SHARED_TENSOR(std::move(ret))};
+ case LossType::LOSS_ENTROPY_SOFTMAX:
+ ret = in[0]->apply(ActivationLayer::softmax);
+ return {MAKE_SHARED_TENSOR(std::move(ret))};
+ case LossType::LOSS_ENTROPY:
+ throw std::runtime_error(
+ "Error: Cross Entropy not supported without softmax or sigmoid.");
+ case LossType::LOSS_UNKNOWN:
+ /** intended */
+ default:
+ throw std::runtime_error("Error: Unknown loss_type.");
+ }
+}
+
+void LossLayer::updateLoss(const Tensor &l) {
+ float loss_sum = 0.0f;
+ const float *data = l.getData();
+
+ for (unsigned int i = 0; i < l.batch(); i++) {
+ loss_sum += data[i];
+ }
+ loss = loss_sum / (float)l.batch();
+}
+
+void LossLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<LossLayer> from = std::static_pointer_cast<LossLayer>(l);
+ this->loss_type = from->loss_type;
+}
+
+sharedConstTensors LossLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ Tensor ret_derivative;
+ Tensor y2 = *derivative[0];
+ Tensor y = input;
+
+ switch (loss_type) {
+ case LossType::LOSS_MSE:
+ ret_derivative = y.subtract(y2).multiply(2).divide(y.getDim().getDataLen());
+ break;
+ case LossType::LOSS_ENTROPY_SIGMOID:
+ y = y.apply(ActivationLayer::sigmoid);
+ ret_derivative = y.subtract(y2).divide(y.getDim().getDataLen());
+ break;
+ case LossType::LOSS_ENTROPY_SOFTMAX:
+ y = y.apply(ActivationLayer::softmax);
+ ret_derivative = y.subtract(y2).divide(y.batch());
+ break;
+ case LossType::LOSS_ENTROPY:
+ throw std::runtime_error(
+ "Error: Cross Entropy not supported without softmax or sigmoid.");
+ case LossType::LOSS_UNKNOWN:
+ /** intended */
+ default:
+ throw std::runtime_error("Unknown loss_type.");
+ }
+
+ return {MAKE_SHARED_TENSOR(std::move(ret_derivative))};
+}
+
+int LossLayer::setLoss(LossType l) {
+ int status = ML_ERROR_NONE;
+ if (l == LossType::LOSS_UNKNOWN) {
+ ml_loge("Error: Unknown loss type");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ loss_type = l;
+ return status;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file loss_layer.h
+ * @date 12 June 2020
+ * @brief This is Loss Layer Class of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __LOSS_LAYER_H__
+#define __LOSS_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Enumeration of loss function type
+ */
+enum class LossType {
+ LOSS_MSE, /** Mean Squared Error */
+ LOSS_ENTROPY, /** Cross Entropy */
+ LOSS_ENTROPY_SIGMOID, /** Cross Entropy amalgamated with sigmoid for stability
+ */
+ LOSS_ENTROPY_SOFTMAX, /** Cross Entropy amalgamated with softmax for stability
+ */
+ LOSS_UNKNOWN /** Unknown */
+};
+
+/**
+ * @class LossLayer
+ * @brief loss layer
+ */
+class LossLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of Loss Layer
+ */
+ template <typename... Args>
+ LossLayer(LossType loss_type_ = LossType::LOSS_UNKNOWN, Args... args) :
+ Layer(LayerType::LAYER_LOSS, args...),
+ loss_type(LossType::LOSS_UNKNOWN) {}
+
+ /**
+ * @brief Destructor of Loss Layer
+ */
+ ~LossLayer(){};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @brief Forward Propagation of a layer
+ * @param[in] in List of Input Tensors taken by this layer
+ * @param[in] label List of Label Tensors for the model
+ * @retval List of Input Tensors as it is.
+ */
+ sharedConstTensors forwarding(sharedConstTensors in,
+ sharedConstTensors label);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @brief read layer Weight & Bias data from file
+ * @param[in] file input file stream
+ */
+ void read(std::ifstream &file) {}
+
+ /**
+ * @brief save layer Weight & Bias data from file
+ * @param[in] file output file stream
+ */
+ void save(std::ofstream &file) {}
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /**
+ * @brief initialize layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Loss"; };
+
+ /**
+ * @brief set loss function
+ * @param[in] l loss type
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setLoss(LossType l);
+
+private:
+ LossType loss_type; /**< loss type of loss layer */
+
+ /**
+ * @brief update loss
+ * @param[in] l Tensor data to calculate
+ */
+ void updateLoss(const Tensor &l);
+};
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __LOSS_LAYER_H__ */
--- /dev/null
+layer_sources = [
+ 'activation_layer.cpp',
+ 'addition_layer.cpp',
+ 'concat_layer.cpp',
+ 'bn_layer.cpp',
+ 'conv2d_layer.cpp',
+ 'fc_layer.cpp',
+ 'flatten_layer.cpp',
+ 'input_layer.cpp',
+ 'layer.cpp',
+ 'layer_factory.cpp',
+ 'loss_layer.cpp',
+ 'pooling2d_layer.cpp'
+]
+
+layer_headers = [
+ 'layer_factory.h',
+ 'layer_internal.h'
+]
+
+layer_deps = []
+
+if get_option('enable-nnstreamer-backbone')
+ if not nnstreamer_capi_dep.found()
+ error('NNStreamer CAPI dependency not found')
+ endif
+ layer_deps += nnstreamer_capi_dep
+ layer_sources += 'nnstreamer_layer.cpp'
+endif
+
+if get_option('enable-tflite-backbone')
+ if not tflite_dep.found()
+ error('Tensorflow-Lite dependency not found')
+ endif
+ layer_deps += tflite_dep
+ layer_sources += 'tflite_layer.cpp'
+endif
+
+nntrainer_base_deps += layer_deps
+
+foreach s : layer_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
+
+foreach h : layer_headers
+ nntrainer_headers += join_paths(meson.current_source_dir(), h)
+endforeach
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file nnstreamer_layer.cpp
+ * @date 26 October 2020
+ * @brief This is class to encapsulate nnstreamer as a layer of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ * @todo: provide input/output dimensions to nnstreamer for certain frameworks
+ * @todo: support transposing the data to support NCHW nntrainer data to NHWC
+ * nnstreamer data
+ */
+
+#include <lazy_tensor.h>
+#include <nnstreamer_layer.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int NNStreamerLayer::nnst_info_to_tensor_dim(ml_tensors_info_h &out_res,
+ TensorDim &dim) {
+ int status = ML_ERROR_NONE;
+ unsigned int count;
+ ml_tensor_type_e type;
+ ml_tensor_dimension dim_;
+
+ status = ml_tensors_info_get_count(out_res, &count);
+ if (status != ML_ERROR_NONE)
+ return status;
+
+ if (count > 1)
+ return ML_ERROR_NOT_SUPPORTED;
+
+ status = ml_tensors_info_get_tensor_type(out_res, 0, &type);
+ if (status != ML_ERROR_NONE)
+ return status;
+
+ if (type != ML_TENSOR_TYPE_FLOAT32)
+ return ML_ERROR_NOT_SUPPORTED;
+
+ if (ML_TENSOR_RANK_LIMIT > MAXDIM)
+ return ML_ERROR_NOT_SUPPORTED;
+
+ status = ml_tensors_info_get_tensor_dimension(out_res, 0, dim_);
+ if (status != ML_ERROR_NONE)
+ return status;
+
+ for (size_t i = 0; i < MAXDIM; i++)
+ dim.setTensorDim(i, dim_[i]);
+
+ /* reverse the dimension as nnstreamer stores dimension in reverse way */
+ dim.reverse();
+
+ return status;
+}
+
+NNStreamerLayer::~NNStreamerLayer() { finalizeError(ML_ERROR_NONE); }
+
+int NNStreamerLayer::finalizeError(int status) {
+ if (in_res) {
+ ml_tensors_info_destroy(in_res);
+ in_res = nullptr;
+ }
+
+ if (out_res) {
+ ml_tensors_info_destroy(out_res);
+ out_res = nullptr;
+ }
+
+ if (in_data_cont) {
+ ml_tensors_data_destroy(in_data_cont);
+ in_data_cont = nullptr;
+ }
+
+ if (out_data_cont) {
+ ml_tensors_data_destroy(out_data_cont);
+ out_data_cont = nullptr;
+ }
+
+ if (single) {
+ ml_single_close(single);
+ single = nullptr;
+ }
+
+ return status;
+}
+
+int NNStreamerLayer::initialize() {
+ int status = ML_ERROR_NONE;
+ TensorDim in_dim;
+
+ status = ml_single_open(&single, modelfile.c_str(), NULL, NULL,
+ ML_NNFW_TYPE_TENSORFLOW_LITE, ML_NNFW_HW_ANY);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ /* input tensor in filter */
+ status = ml_single_get_input_info(single, &in_res);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ status = nnst_info_to_tensor_dim(in_res, in_dim);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ if (input_dim[0].getTensorDim(0) != 0 && input_dim[0] != in_dim) {
+ ml_loge("Set tensor info does not match info from the framework");
+ return finalizeError(ML_ERROR_INVALID_PARAMETER);
+ } else {
+ input_dim[0] = in_dim;
+ }
+
+ /* input tensor in filter */
+ status = ml_single_get_output_info(single, &out_res);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ status = nnst_info_to_tensor_dim(out_res, output_dim[0]);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ /* generate input data container */
+ status = ml_tensors_data_create(in_res, &in_data_cont);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ size_t in_data_size;
+ status =
+ ml_tensors_data_get_tensor_data(in_data_cont, 0, &in_data, &in_data_size);
+ if (status != ML_ERROR_NONE)
+ return finalizeError(status);
+
+ if (in_data_size != input_dim[0].getDataLen() * sizeof(float))
+ return finalizeError(ML_ERROR_INVALID_PARAMETER);
+
+ return status;
+}
+
+void NNStreamerLayer::setTrainable(bool train) {
+ if (train)
+ throw exception::not_supported(
+ "NNStreamer layer does not support training");
+
+ Layer::setTrainable(false);
+}
+
+void NNStreamerLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ switch (type) {
+ case PropertyType::modelfile: {
+ if (!value.empty())
+ modelfile = value;
+ } break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+sharedConstTensors NNStreamerLayer::forwarding(sharedConstTensors in) {
+ size_t data_size;
+ input = *in[0];
+
+ std::copy(input.getData(), input.getData() + input.length(),
+ (float *)in_data);
+
+ int status = ml_single_invoke(single, in_data_cont, &out_data_cont);
+ if (status != ML_ERROR_NONE)
+ throw std::runtime_error("Failed to forward nnstreamer backbone");
+
+ hidden = Tensor(output_dim[0], static_cast<const float *>(out_data));
+ status =
+ ml_tensors_data_get_tensor_data(out_data_cont, 0, &out_data, &data_size);
+ if (status != ML_ERROR_NONE) {
+ ml_tensors_data_destroy(out_data_cont);
+ out_data_cont = nullptr;
+ throw std::runtime_error("Failed to forward nnstreamer backbone");
+ }
+
+ if (data_size != hidden.getSize()) {
+ ml_tensors_data_destroy(out_data_cont);
+ out_data_cont = nullptr;
+ throw std::runtime_error("Output size mismatch from nnstreamer backbone.");
+ }
+
+ std::copy((float *)out_data, (float *)((char *)out_data + data_size),
+ hidden.getData());
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+void NNStreamerLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<NNStreamerLayer> from =
+ std::static_pointer_cast<NNStreamerLayer>(l);
+ this->modelfile = from->modelfile;
+}
+
+sharedConstTensors NNStreamerLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ throw exception::not_supported(
+ "Backwarding is not supported for nnstreamer layer");
+}
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file nnstreamer_layer.h
+ * @date 26 October 2020
+ * @brief This is class to encapsulate nnstreamer as a layer of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug no known bugs except for NYI items
+ *
+ */
+
+#ifndef __NNSTREAMER_LAYER_H__
+#define __NNSTREAMER_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <nnstreamer-single.h>
+#include <nnstreamer.h>
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @class NNStreamerLayer
+ * @brief nnstreamer layer
+ */
+class NNStreamerLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of NNStreamer Layer
+ */
+ NNStreamerLayer(std::string model = "") :
+ Layer(LayerType::LAYER_BACKBONE_NNSTREAMER),
+ modelfile(model),
+ single(nullptr),
+ in_res(nullptr),
+ out_res(nullptr),
+ in_data_cont(nullptr),
+ out_data_cont(nullptr) {
+ trainable = false;
+ }
+
+ /**
+ * @brief Destructor of NNStreamer Layer
+ */
+ ~NNStreamerLayer();
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @copydoc Layer::copy(std::shared_ptr<layer> l)
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /**
+ * @copydoc Layer::initialize()
+ */
+ int initialize();
+
+ /**
+ * @copydoc Layer::setTrainable(bool train)
+ */
+ void setTrainable(bool train);
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "BackboneNNStreamer"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ std::string modelfile;
+ ml_single_h single;
+ ml_tensors_info_h in_res, out_res;
+ ml_tensors_data_h in_data_cont, out_data_cont;
+ void *in_data, *out_data;
+
+ /**
+ * @brief finalize the layer with the given status
+ * @param[in] status status to return
+ * @retval return status received as argument
+ */
+ int finalizeError(int status);
+
+ /**
+ * @brief convert nnstreamer's tensor_info to nntrainer's tensor_dim
+ * @param[in] out_res nnstreamer's tensor_info
+ * @param[out] dim nntrainer's tensor_dim
+ * @retval 0 on success, -errno on failure
+ */
+ static int nnst_info_to_tensor_dim(ml_tensors_info_h &out_res,
+ TensorDim &dim);
+};
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __NNSTREAMER_LAYER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file pooling2d_layer.cpp
+ * @date 12 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is 2 Dimensional Pooling Layer Class for Neural Network
+ *
+ */
+
+#include <cstring>
+#include <limits>
+
+#include <layer_internal.h>
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <pooling2d_layer.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int Pooling2DLayer::initialize() {
+ int status = ML_ERROR_NONE;
+
+ if (input_dim.size() != 1 || output_dim.size() != 1) {
+ throw std::invalid_argument("Convolution layer only takes one input");
+ }
+
+ TensorDim &in_dim = input_dim[0];
+ TensorDim &out_dim = output_dim[0];
+
+ if (in_dim.getDataLen() == 1) {
+ ml_logw("Warning: the length of previous layer dimension is one");
+ }
+
+ out_dim.batch(in_dim.batch());
+ out_dim.channel(in_dim.channel());
+
+ if (pooling_type == PoolingType::max ||
+ pooling_type == PoolingType::average) {
+ out_dim.height(
+ (in_dim.height() - pool_size[0] + 2 * padding[0]) / stride[0] + 1);
+ out_dim.width((in_dim.width() - pool_size[1] + 2 * padding[1]) / stride[1] +
+ 1);
+ } else {
+ out_dim.height(1);
+ out_dim.width(1);
+ }
+
+ if (pooling_type == PoolingType::max) {
+ max_idx.resize(out_dim.getDataLen());
+ }
+
+ if (pooling_type == PoolingType::global_max) {
+ max_idx_global.resize(out_dim.getDataLen());
+ }
+
+ return status;
+}
+
+sharedConstTensors Pooling2DLayer::forwarding(sharedConstTensors in) {
+ input = *in[0];
+
+ TensorDim hidden_dim = output_dim[0];
+ TensorDim &in_dim = input_dim[0];
+
+ hidden = Tensor(hidden_dim);
+ hidden.setZero();
+
+ for (unsigned int b = 0; b < in_dim.batch(); ++b) {
+ Tensor in_padded = zero_pad(b, input, padding.data());
+ Tensor result = pooling2d(b, in_padded);
+ memcpy(hidden.getAddress(b * hidden.getDim().getFeatureLen()),
+ result.getData(), result.getDim().getDataLen() * sizeof(float));
+ }
+
+ return {MAKE_SHARED_TENSOR(hidden)};
+}
+
+sharedConstTensors Pooling2DLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ unsigned int batch = input_dim[0].batch();
+ unsigned int channel = input_dim[0].channel();
+ unsigned int height = input_dim[0].height();
+ unsigned int width = input_dim[0].width();
+ unsigned int p_height = pool_size[0];
+ unsigned int p_width = pool_size[1];
+ unsigned int p_size = p_height * p_width;
+
+ unsigned int J, K;
+ Tensor result = Tensor(input_dim[0]);
+ result.setZero();
+ float *out = result.getData();
+ switch (pooling_type) {
+ case PoolingType::max: {
+ for (unsigned int i = 0; i < derivative[0]->getDim().getDataLen(); ++i) {
+ out[max_idx[i]] += derivative[0]->getData()[i];
+ }
+ } break;
+ case PoolingType::average: {
+ for (unsigned int b = 0; b < batch; ++b) {
+ for (unsigned int i = 0; i < channel; ++i) {
+ J = 0;
+ for (unsigned int j = 0; j <= height - p_height; j += stride[0]) {
+ K = 0;
+ for (unsigned int k = 0; k <= width - p_width; k += stride[1]) {
+ float del =
+ derivative[0]->getValue(b, i, J, K) / static_cast<float>(p_size);
+ for (unsigned int pi = 0; pi < p_height; ++pi) {
+ for (unsigned int pj = 0; pj < p_width; ++pj) {
+ result.setValue(b, i, j + pi, k + pj,
+ result.getValue(b, i, j + pi, k + pj) + del);
+ }
+ }
+ K++;
+ }
+ J++;
+ }
+ }
+ }
+ } break;
+ case PoolingType::global_max: {
+ for (unsigned int i = 0; i < derivative[0]->getDim().getDataLen(); ++i) {
+ float der = derivative[0]->getData()[i] / max_idx_global[i].size();
+ for (unsigned int m = 0; m < max_idx_global[i].size(); m++) {
+ out[max_idx_global[i][m]] += der;
+ }
+ }
+ } break;
+ case PoolingType::global_average: {
+ unsigned int p_size = width * height;
+ for (unsigned int b = 0; b < batch; ++b) {
+ for (unsigned int i = 0; i < channel; ++i) {
+ float del = derivative[0]->getValue(b, i, 0, 0) / (p_size);
+ for (unsigned int j = 0; j < height; ++j) {
+ for (unsigned int k = 0; k < width; ++k) {
+ result.setValue(b, i, j, k, del);
+ }
+ }
+ }
+ }
+
+ } break;
+ default:
+ throw std::runtime_error("Error: Unknown Pooling Type");
+ }
+ return {MAKE_SHARED_TENSOR(std::move(result))};
+}
+
+int Pooling2DLayer::setSize(int *size, PropertyType type) {
+ int status = ML_ERROR_NONE;
+ switch (type) {
+ case PropertyType::pool_size:
+ for (int i = 0; i < POOLING2D_DIM; ++i) {
+ pool_size[i] = size[i];
+ }
+ break;
+ case PropertyType::stride:
+ for (int i = 0; i < POOLING2D_DIM; ++i) {
+ stride[i] = size[i];
+ }
+ break;
+ case PropertyType::padding:
+ for (int i = 0; i < POOLING2D_DIM; ++i) {
+ padding[i] = size[i];
+ }
+ break;
+ default:
+ ml_loge("Error: Unknown Layer Property type");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+ return status;
+}
+
+void Pooling2DLayer::setBatch(unsigned int batch) {
+ Layer::setBatch(batch);
+
+ if (pooling_type == PoolingType::max) {
+ max_idx.resize(output_dim[0].getDataLen());
+ } else if (pooling_type == PoolingType::global_max) {
+ max_idx_global.resize(output_dim[0].getDataLen());
+ }
+}
+
+void Pooling2DLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<Pooling2DLayer> from =
+ std::static_pointer_cast<Pooling2DLayer>(l);
+
+ this->pooling_type = from->pooling_type;
+
+ for (unsigned int i = 0; i < POOLING2D_DIM; ++i) {
+ this->pool_size[i] = from->pool_size[i];
+ this->stride[i] = from->stride[i];
+ this->padding[i] = from->padding[i];
+ }
+}
+
+void Pooling2DLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::pooling:
+ if (!value.empty()) {
+ pooling_type = (PoolingType)parseType(value, TOKEN_POOLING);
+ if (pooling_type == PoolingType::unknown) {
+ throw std::invalid_argument("[Pooling2d_layer]: Unknown pooling type");
+ }
+ break;
+ }
+ case PropertyType::pool_size:
+ if (!value.empty()) {
+ status = getValues(POOLING2D_DIM, value, (int *)(pool_size.data()));
+ throw_status(status);
+ if (pool_size[0] == 0 || pool_size[1] == 0) {
+ throw std::invalid_argument(
+ "[Pooling2d_layer] pool_size must be greater than 0");
+ }
+ }
+ break;
+ case PropertyType::stride:
+ if (!value.empty()) {
+ status = getValues(POOLING2D_DIM, value, (int *)(stride.data()));
+ throw_status(status);
+ if (stride[0] == 0 || stride[1] == 0) {
+ throw std::invalid_argument(
+ "[Pooling2d_layer] stride must be greater than 0");
+ }
+ }
+ break;
+ case PropertyType::padding:
+ if (!value.empty()) {
+ status = getValues(POOLING2D_DIM, value, (int *)(padding.data()));
+ throw_status(status);
+ if ((int)padding[0] < 0 || (int)padding[1] < 0) {
+ throw std::invalid_argument(
+ "[Pooling2d_layer] padding must be greater than 0");
+ }
+ }
+ break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+Tensor Pooling2DLayer::pooling2d(unsigned int batch, Tensor &in) {
+ unsigned int channel = in.channel();
+ unsigned int height = in.height();
+ unsigned int width = in.width();
+ unsigned int p_height = pool_size[0];
+ unsigned int p_width = pool_size[1];
+ TensorDim &out_dim = output_dim[0];
+ unsigned int base_idx = batch * out_dim.getFeatureLen();
+
+ Tensor output(out_dim.channel(), out_dim.height(), out_dim.width());
+
+ unsigned int J, K;
+ switch (pooling_type) {
+ case PoolingType::max: {
+ for (unsigned int i = 0; i < channel; ++i) {
+ J = 0;
+ for (unsigned int j = 0; j <= height - p_height; j += stride[0]) {
+ K = 0;
+ for (unsigned int k = 0; k <= width - p_width; k += stride[1]) {
+ float max = std::numeric_limits<float>::lowest();
+ for (unsigned int pi = 0; pi < p_height; ++pi) {
+ for (unsigned int pj = 0; pj < p_width; ++pj) {
+ float val = in.getValue(0, i, j + pi, k + pj);
+ if (max < val) {
+ max_idx[base_idx + i * out_dim.height() * out_dim.width() +
+ J * out_dim.width() + K] =
+ batch * input_dim[0].getFeatureLen() + i * height * width +
+ (j + pi) * width + (k + pj);
+ max = val;
+ }
+ }
+ }
+ output.setValue(0, i, J, K, max);
+ K++;
+ }
+ J++;
+ }
+ }
+ } break;
+ case PoolingType::average: {
+ unsigned int p_size = p_height * p_width;
+ for (unsigned int i = 0; i < channel; ++i) {
+ J = 0;
+ for (unsigned int j = 0; j <= height - p_height; j += stride[0]) {
+ K = 0;
+ for (unsigned int k = 0; k <= width - p_width; k += stride[1]) {
+ float sum = 0.0f;
+ for (unsigned int pi = 0; pi < p_height; ++pi) {
+ for (unsigned int pj = 0; pj < p_width; ++pj) {
+ sum += in.getValue(0, i, j + pi, k + pj);
+ }
+ }
+ sum = sum / static_cast<float>(p_size);
+ output.setValue(0, i, J, K, sum);
+ K++;
+ }
+ J++;
+ }
+ }
+ } break;
+ case PoolingType::global_max: {
+ output.setZero();
+ for (unsigned int i = 0; i < channel; ++i) {
+ unsigned int idx =
+ batch * input_dim[0].getFeatureLen() + i * height * width;
+ float max = std::numeric_limits<float>::lowest();
+ max_idx_global[base_idx + i].clear();
+ for (unsigned int j = 0; j < height; ++j) {
+ for (unsigned int k = 0; k < width; ++k) {
+ float val = in.getValue(0, i, j, k);
+ if (max <= val) {
+ if (max < val)
+ max_idx_global[base_idx + i].clear();
+ max_idx_global[base_idx + i].push_back(idx + j * width + k);
+ max = val;
+ }
+ }
+ }
+ output.setValue(0, i, 0, 0, max);
+ }
+ } break;
+ case PoolingType::global_average: {
+ output.setZero();
+ Tensor sum_wh = in.chain().sum(3).sum(2).run();
+ for (unsigned int i = 0; i < channel; ++i) {
+ output.setValue(0, i, 0, 0,
+ sum_wh.getValue(0, i, 0, 0) / (in.width() * in.height()));
+ }
+ } break;
+ default:
+ ml_loge("Error: Unknown Pooling Type");
+ throw std::runtime_error("Error: Unknown Pooling Type");
+ break;
+ }
+
+ return output;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file pooling2d_layer.h
+ * @date 12 June 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is 2 Dimensional Pooling Layer Class for Neural Network
+ *
+ */
+
+#ifndef __POOLING2D_LAYER_H__
+#define __POOLING2D_LAYER_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+#include <vector>
+
+#define POOLING2D_DIM 2
+
+namespace nntrainer {
+
+/**
+ * @class Pooling 2D Layer
+ * @brief Pooling 2D Layer
+ */
+class Pooling2DLayer : public Layer {
+public:
+ enum class PoolingType {
+ max = 0,
+ average = 1,
+ global_max = 2,
+ global_average = 3,
+ unknown = 4,
+ };
+
+ /**
+ * @brief Constructor of Pooling 2D Layer
+ */
+ template <typename... Args>
+ Pooling2DLayer(
+ PoolingType pooling_type_ = PoolingType::average,
+ const std::array<unsigned int, POOLING2D_DIM> &pool_size_ = {0, 0},
+ const std::array<unsigned int, POOLING2D_DIM> &stride_ = {1, 1},
+ const std::array<unsigned int, POOLING2D_DIM> &padding_ = {0, 0},
+ Args... args) :
+ Layer(LayerType::LAYER_POOLING2D, args...),
+ pool_size(pool_size_),
+ stride(stride_),
+ padding(padding_),
+ pooling_type(pooling_type_) {}
+
+ /**
+ * @brief Destructor of Pooling 2D Layer
+ */
+ ~Pooling2DLayer() {}
+
+ /**
+ * @brief Move constructor of Pooling 2D Layer.
+ * @param[in] Pooling2D &&
+ */
+ Pooling2DLayer(Pooling2DLayer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs Pooling2DLayer to be moved.
+ */
+ Pooling2DLayer &operator=(Pooling2DLayer &&rhs) = default;
+
+ /**
+ * @brief initialize layer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int initialize();
+
+ /**
+ * @brief Read Weight & Bias Data from file
+ * @param[in] file input stream file
+ */
+ void read(std::ifstream &file){};
+
+ /**
+ * @brief Save Weight & Bias Data to file
+ * @param[in] file output stream file
+ */
+ void save(std::ofstream &file){};
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @copydoc Layer::setBatch(unsigned int batch)
+ */
+ void setBatch(unsigned int batch);
+
+ /**
+ * @brief copy layer
+ * @param[in] l layer to copy
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /* TO DO : support keras type of padding */
+ enum class PaddingType {
+ full = 0,
+ same = 1,
+ valid = 2,
+ unknown = 3,
+ };
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "Pooling2D"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ std::array<unsigned int, POOLING2D_DIM> pool_size;
+ std::array<unsigned int, POOLING2D_DIM> stride;
+ std::array<unsigned int, POOLING2D_DIM> padding;
+ std::vector<unsigned int> max_idx;
+ std::vector<std::vector<unsigned int>> max_idx_global;
+ PoolingType pooling_type;
+
+ /**
+ * @brief calculation convolution
+ * @param[in] batch batch index
+ * @param[in] in input tensor
+ * @retval Tensor outoput tensor
+ */
+ Tensor pooling2d(unsigned int batch, Tensor &in);
+
+ /**
+ * @brief set Pooling Type
+ * @param[in] t pooling type
+ */
+ void setPoolingType(PoolingType t) { pooling_type = t; };
+
+ /**
+ * @brief set Parameter Size
+ * @param[in] * size : size arrary
+ * @param[in] type : Property type
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setSize(int *size, PropertyType type);
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __POOLING_LAYER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file tflite_layer.cpp
+ * @date 26 October 2020
+ * @brief This is class to encapsulate tflite as a layer of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <tflite_layer.h>
+
+namespace nntrainer {
+
+void TfLiteLayer::setDimensions(const std::vector<int> &tensor_idx_list,
+ std::vector<TensorDim> &dim, bool is_output) {
+ unsigned int num_tensors = tensor_idx_list.size();
+ dim.resize(num_tensors);
+
+ for (unsigned int i = 0; i < num_tensors; i++) {
+ unsigned int tensor_idx = tensor_idx_list[i];
+ if (is_output && interpreter->tensor(tensor_idx)->type != kTfLiteFloat32)
+ throw exception::not_supported(
+ "Data type other than float32 not supported");
+
+ unsigned int num_dims = interpreter->tensor(tensor_idx)->dims->size;
+ if (num_dims > MAXDIM)
+ throw exception::not_supported("Number of dimensions exceed the support");
+
+ /** This puts the unused dimensions to the outer dimensions */
+ for (size_t dim_idx = 0; dim_idx < num_dims; dim_idx++)
+ dim[i].setTensorDim(
+ MAXDIM - dim_idx - 1,
+ interpreter->tensor(tensor_idx)->dims->data[num_dims - dim_idx - 1]);
+ }
+}
+
+int TfLiteLayer::initialize() {
+ tflite::ops::builtin::BuiltinOpResolver resolver;
+
+ model = tflite::FlatBufferModel::BuildFromFile(modelfile.c_str());
+ if (!model)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ tflite::InterpreterBuilder(*model.get(), resolver)(&interpreter);
+ if (!interpreter)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ if (interpreter->AllocateTensors() != kTfLiteOk)
+ throw std::runtime_error("Failed to allocate tensors!");
+
+ std::vector<TensorDim> dims;
+ setDimensions(interpreter->inputs(), dims, false);
+ setDimensions(interpreter->outputs(), output_dim, true);
+
+ if (input_dim.size() && input_dim[0].getTensorDim(0) != 0) {
+ if (dims.size() != input_dim.size())
+ throw std::invalid_argument(
+ "Provided number of input dimensions mismatch");
+
+ for (size_t idx = 0; idx < dims.size(); idx++) {
+ if (dims[idx] != input_dim[idx])
+ throw std::invalid_argument("Input dimensions mismatch");
+ }
+ } else {
+ input_dim.resize(dims.size());
+ std::copy(dims.begin(), dims.end(), input_dim.begin());
+ }
+
+ return ML_ERROR_NONE;
+}
+
+void TfLiteLayer::setTrainable(bool train) {
+ if (train)
+ throw exception::not_supported("TfLite layer does not support training");
+
+ Layer::setTrainable(false);
+}
+
+void TfLiteLayer::setProperty(const PropertyType type,
+ const std::string &value) {
+ switch (type) {
+ case PropertyType::modelfile: {
+ if (!value.empty())
+ modelfile = value;
+ } break;
+ default:
+ Layer::setProperty(type, value);
+ break;
+ }
+}
+
+sharedConstTensors TfLiteLayer::forwarding(sharedConstTensors in) {
+#ifdef DEBUG
+ std::vector<TensorDim> dims;
+ if (in.size() != input_dim.size())
+ throw std::invalid_argument("Provided number of input dimensions mismatch");
+
+ for (int idx = 0; idx < dims.size(); idx++) {
+ if (in[idx].getDim() != input_dim[idx])
+ throw std::invalid_argument("Input dimensions mismatch");
+ }
+#endif
+
+ sharedConstTensors out;
+
+ auto in_indices = interpreter->inputs();
+ for (size_t idx = 0; idx < in.size(); idx++)
+ interpreter->tensor(in_indices[idx])->data.raw = (char *)in[idx]->getData();
+
+ auto out_indices = interpreter->outputs();
+ out.resize(out_indices.size());
+ for (size_t idx = 0; idx < out_indices.size(); idx++) {
+ out[idx] = MAKE_SHARED_TENSOR(output_dim[idx]);
+ interpreter->tensor(out_indices[idx])->data.raw =
+ (char *)out[idx]->getData();
+ }
+
+ int status = interpreter->Invoke();
+ if (status != kTfLiteOk)
+ throw std::runtime_error("Invoke failed");
+
+ hidden = *out[0];
+
+ return out;
+}
+
+void TfLiteLayer::copy(std::shared_ptr<Layer> l) {
+ Layer::copy(l);
+
+ std::shared_ptr<TfLiteLayer> from = std::static_pointer_cast<TfLiteLayer>(l);
+ this->modelfile = from->modelfile;
+}
+
+sharedConstTensors TfLiteLayer::backwarding(sharedConstTensors derivative,
+ int iteration) {
+ throw exception::not_supported(
+ "Backwarding is not supported for tflite layer");
+}
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file tflite_layer.h
+ * @date 3 November 2020
+ * @brief This is class to encapsulate tflite as a layer of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __TENSORFLOW_LITE_H__
+#define __TENSORFLOW_LITE_H__
+#ifdef __cplusplus
+
+#include <layer_internal.h>
+#include <tensor.h>
+
+#include <tensorflow/contrib/lite/interpreter.h>
+#include <tensorflow/contrib/lite/kernels/register.h>
+#include <tensorflow/contrib/lite/model.h>
+
+namespace nntrainer {
+
+/**
+ * @class TfLiteLayer
+ * @brief Tensorflow Lite layer
+ */
+class TfLiteLayer : public Layer {
+public:
+ /**
+ * @brief Constructor of NNStreamer Layer
+ */
+ TfLiteLayer(std::string model = "") :
+ Layer(LayerType::LAYER_BACKBONE_TFLITE),
+ modelfile(model),
+ interpreter(nullptr),
+ model(nullptr) {
+ trainable = false;
+ }
+
+ /**
+ * @brief Destructor of NNStreamer Layer
+ */
+ ~TfLiteLayer() = default;
+
+ /**
+ * @copydoc Layer::forwarding(sharedConstTensors in)
+ */
+ sharedConstTensors forwarding(sharedConstTensors in);
+
+ /**
+ * @copydoc Layer::backwarding(sharedConstTensors in, int iteration)
+ */
+ sharedConstTensors backwarding(sharedConstTensors in, int iteration);
+
+ /**
+ * @copydoc Layer::copy(std::shared_ptr<layer> l)
+ */
+ void copy(std::shared_ptr<Layer> l);
+
+ /**
+ * @copydoc Layer::initialize()
+ */
+ int initialize();
+
+ /**
+ * @copydoc Layer::setTrainable(bool train)
+ */
+ void setTrainable(bool train);
+
+ /**
+ * @brief get the base name for the layer
+ * @retval base name of the layer
+ */
+ std::string getBaseName() { return "BackboneTFLite"; };
+
+ using Layer::setProperty;
+
+ /**
+ * @copydoc Layer::setProperty(const PropertyType type, const std::string
+ * &value)
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+private:
+ std::string modelfile;
+ std::unique_ptr<tflite::Interpreter> interpreter;
+ std::unique_ptr<tflite::FlatBufferModel> model;
+
+ void setDimensions(const std::vector<int> &tensor_idx_list,
+ std::vector<TensorDim> &dim, bool is_output);
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __TENSORFLOW_LITE_H__ */
nntrainer_inc = [
- include_directories('./include'),
- include_directories('../api'),
- include_directories('../api/ccapi/include')
+ include_directories('.'),
+ include_directories('../api', '../api/ccapi/include')
]
+nntrainer_sources = []
+nntrainer_headers = []
+
# pc file is not present for 'ml-api-common' yet
if cxx.has_header('nnstreamer/ml-api-common.h', required: false)
nntrainer_inc += include_directories ('/usr/include/nnstreamer')
nntrainer_base_deps += dependency('dlog')
endif
-nntrainer_sources = [
- 'src/activation_layer.cpp',
- 'src/addition_layer.cpp',
- 'src/concat_layer.cpp',
- 'src/blas_interface.cpp',
- 'src/bn_layer.cpp',
- 'src/conv2d_layer.cpp',
- 'src/databuffer.cpp',
- 'src/databuffer_factory.cpp',
- 'src/databuffer_file.cpp',
- 'src/databuffer_func.cpp',
- 'src/fc_layer.cpp',
- 'src/flatten_layer.cpp',
- 'src/input_layer.cpp',
- 'src/layer.cpp',
- 'src/layer_factory.cpp',
- 'src/lazy_tensor.cpp',
- 'src/loss_layer.cpp',
- 'src/model_loader.cpp',
- 'src/neuralnet.cpp',
- 'src/nntrainer_logger.cpp',
- 'src/optimizer.cpp',
- 'src/optimizer_factory.cpp',
- 'src/parse_util.cpp',
- 'src/pooling2d_layer.cpp',
- 'src/sgd.cpp',
- 'src/adam.cpp',
- 'src/tensor.cpp',
- 'src/tensor_dim.cpp',
- 'src/util_func.cpp',
- 'src/weight.cpp'
+nntrainer_elements = [
+ 'dataset',
+ 'layers',
+ 'models',
+ 'optimizers',
+ 'tensor',
+ 'utils',
]
-nntrainer_headers = [
- 'include/activation_layer.h',
- 'include/addition_layer.h',
- 'include/concat_layer.h',
- 'include/blas_interface.h',
- 'include/bn_layer.h',
- 'include/conv2d_layer.h',
- 'include/databuffer.h',
- 'include/databuffer_factory.h',
- 'include/databuffer_file.h',
- 'include/databuffer_func.h',
- 'include/databuffer_util.h',
- 'include/delegate.h',
- 'include/fc_layer.h',
- 'include/flatten_layer.h',
- 'include/input_layer.h',
- 'include/layer_internal.h',
- 'include/layer_factory.h',
- 'include/lazy_tensor.h',
- 'include/loss_layer.h',
- 'include/model_loader.h',
- 'include/neuralnet.h',
- 'include/nntrainer_log.h',
- 'include/nntrainer_logger.h',
- 'include/optimizer_internal.h',
- 'include/parse_util.h',
- 'include/pooling2d_layer.h',
- 'include/sgd.h',
- 'include/adam.h',
- 'include/tensor.h',
- 'include/tensor_dim.h',
- 'include/util_func.h',
- 'include/weight.h',
- 'include/optimizer_factory.h',
-]
+foreach elem : nntrainer_elements
+ subdir(elem)
+ nntrainer_inc += include_directories(elem)
+endforeach
-if get_option('enable-nnstreamer-backbone')
- if not nnstreamer_capi_dep.found()
- error('NNStreamer CAPI dependency not found')
- endif
- nntrainer_base_deps += nnstreamer_capi_dep
- nntrainer_headers += 'include/nnstreamer_layer.h'
- nntrainer_sources += 'src/nnstreamer_layer.cpp'
-endif
+nntrainer_common_sources = [
+ 'nntrainer_logger.cpp',
+]
-if get_option('enable-tflite-backbone')
- if not tflite_dep.found()
- error('Tensorflow-Lite dependency not found')
- endif
- nntrainer_base_deps += tflite_dep
- nntrainer_headers += 'include/tflite_layer.h'
- nntrainer_sources += 'src/tflite_layer.cpp'
-endif
+foreach s : nntrainer_common_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
# Build libraries
nntrainer_shared = shared_library('nntrainer',
install_headers(nntrainer_headers,
subdir: 'nntrainer'
)
+
--- /dev/null
+model_sources = [
+ 'model_loader.cpp',
+ 'neuralnet.cpp'
+]
+
+model_headers = [
+ 'neuralnet.h'
+]
+
+foreach s : model_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
+
+foreach h : model_headers
+ nntrainer_headers += join_paths(meson.current_source_dir(), h)
+endforeach
+
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file model_loader.c
+ * @date 5 August 2020
+ * @brief This is model loader class for the Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <adam.h>
+#include <databuffer_factory.h>
+#include <databuffer_file.h>
+#include <databuffer_func.h>
+#include <layer_factory.h>
+#include <model_loader.h>
+#include <neuralnet.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <optimizer_factory.h>
+#include <parse_util.h>
+#include <sstream>
+#include <util_func.h>
+
+#define NN_INI_RETURN_STATUS() \
+ do { \
+ if (status != ML_ERROR_NONE) { \
+ iniparser_freedict(ini); \
+ return status; \
+ } \
+ } while (0)
+
+namespace nntrainer {
+
+/**
+ * @brief load model config from ini
+ */
+int ModelLoader::loadModelConfigIni(dictionary *ini, NeuralNetwork &model) {
+ int status = ML_ERROR_NONE;
+
+ if (iniparser_find_entry(ini, "Model") == 0) {
+ ml_loge("there is no [Model] section in given ini file");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ /** Default to neural network model type */
+ model.net_type = (nntrainer::NetType)parseType(
+ iniparser_getstring(ini, "Model:Type", unknown), TOKEN_MODEL);
+ model.epochs = iniparser_getint(ini, "Model:Epochs", model.epochs);
+ model.loss_type = (LossType)parseType(
+ iniparser_getstring(ini, "Model:Loss", unknown), TOKEN_LOSS);
+ model.save_path = iniparser_getstring(ini, "Model:Save_path", "./model.bin");
+ model.batch_size =
+ iniparser_getint(ini, "Model:Batch_Size", model.batch_size);
+
+ /** Default to adam optimizer */
+ OptType opt_type = (OptType)parseType(
+ iniparser_getstring(ini, "Model:Optimizer", "adam"), TOKEN_OPT);
+
+ try {
+ model.opt = nntrainer::createOptimizer(opt_type);
+ } catch (std::exception &e) {
+ ml_loge("%s %s", typeid(e).name(), e.what());
+ return ML_ERROR_INVALID_PARAMETER;
+ } catch (...) {
+ ml_loge("Creating the optimizer failed");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ std::vector<std::string> optimizer_prop = {};
+ optimizer_prop.push_back(
+ {"learning_rate=" +
+ std::string(iniparser_getstring(
+ ini, "Model:Learning_rate",
+ std::to_string(model.opt->getLearningRate()).c_str()))});
+
+ optimizer_prop.push_back(
+ {"decay_steps=" + std::string(iniparser_getstring(
+ ini, "Model:Decay_steps",
+ std::to_string(model.opt->getDecaySteps()).c_str()))});
+ optimizer_prop.push_back(
+ {"decay_rate=" + std::string(iniparser_getstring(
+ ini, "Model:Decay_rate",
+ std::to_string(model.opt->getDecayRate()).c_str()))});
+
+ if (model.opt->getType() == OptType::ADAM) {
+ std::shared_ptr<Adam> opt_adam = std::static_pointer_cast<Adam>(model.opt);
+
+ optimizer_prop.push_back(
+ {"beta1=" +
+ std::string(iniparser_getstring(
+ ini, "Model:Beta1", std::to_string(opt_adam->getBeta1()).c_str()))});
+ optimizer_prop.push_back(
+ {"beta2=" +
+ std::string(iniparser_getstring(
+ ini, "Model:Beta2", std::to_string(opt_adam->getBeta2()).c_str()))});
+ optimizer_prop.push_back(
+ {"epsilon=" + std::string(iniparser_getstring(
+ ini, "Model:Epsilon",
+ std::to_string(opt_adam->getEpsilon()).c_str()))});
+ }
+
+ status = model.opt->setProperty(optimizer_prop);
+ NN_RETURN_STATUS();
+
+ return status;
+}
+
+/**
+ * @brief load dataset config from ini
+ */
+int ModelLoader::loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model) {
+ ml_logd("start parsing dataset config");
+ int status = ML_ERROR_NONE;
+
+ if (iniparser_find_entry(ini, "Dataset") == 0) {
+ model.data_buffer = nntrainer::createDataBuffer(DataBufferType::GENERATOR);
+ status = model.data_buffer->setBatchSize(model.batch_size);
+ return status;
+ }
+
+ if (iniparser_find_entry(ini, "DataSet:Tflite")) {
+ ml_loge("Error: Tflite dataset is not yet implemented!");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ model.data_buffer = nntrainer::createDataBuffer(DataBufferType::FILE);
+ std::shared_ptr<DataBufferFromDataFile> dbuffer =
+ std::static_pointer_cast<DataBufferFromDataFile>(model.data_buffer);
+
+ std::function<int(const char *, DataType, bool)> parse_and_set =
+ [&](const char *key, DataType dt, bool required) -> int {
+ const char *path = iniparser_getstring(ini, key, NULL);
+
+ if (path == NULL) {
+ return required ? ML_ERROR_INVALID_PARAMETER : ML_ERROR_NONE;
+ }
+
+ return dbuffer->setDataFile(dt, path);
+ };
+
+ status = parse_and_set("DataSet:TrainData", DATA_TRAIN, true);
+ NN_RETURN_STATUS();
+ status = parse_and_set("DataSet:ValidData", DATA_VAL, false);
+ NN_RETURN_STATUS();
+ status = parse_and_set("DataSet:TestData", DATA_TEST, false);
+ NN_RETURN_STATUS();
+ status = parse_and_set("Dataset:LabelData", DATA_LABEL, true);
+ NN_RETURN_STATUS();
+
+ status = model.data_buffer->setBatchSize(model.batch_size);
+ NN_RETURN_STATUS();
+
+ unsigned int bufsize = iniparser_getint(ini, "DataSet:BufferSize", 1);
+ status = model.data_buffer->setBufSize(bufsize);
+ NN_RETURN_STATUS();
+
+ ml_logd("parsing dataset done");
+ return status;
+}
+
+int ModelLoader::loadLayerConfigIniCommon(dictionary *ini,
+ std::shared_ptr<Layer> &layer,
+ const std::string &layer_name,
+ LayerType layer_type) {
+ int status = ML_ERROR_NONE;
+
+ try {
+ layer = nntrainer::createLayer(layer_type);
+ } catch (const std::exception &e) {
+ ml_loge("%s %s", typeid(e).name(), e.what());
+ status = ML_ERROR_INVALID_PARAMETER;
+ } catch (...) {
+ ml_loge("unknown error type thrown");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+ NN_RETURN_STATUS();
+
+ unsigned int property_end =
+ static_cast<unsigned int>(Layer::PropertyType::unknown);
+
+ for (unsigned int i = 0; i < property_end; ++i) {
+ std::string prop = propToStr(i);
+ std::string value =
+ iniparser_getstring(ini, (layer_name + ":" + prop).c_str(), unknown);
+
+ /**! @todo: add following negative tc after #319
+ * 1. layer has empty prop -> throw std::invalid_argument
+ * 2. layer has not allowed property -> throw exception::not_supported
+ * 3. property value parse error -> throw std::invalid_argument
+ */
+ if (!strncmp(value.c_str(), unknown, strlen(unknown))) {
+ continue;
+ }
+
+ if (value == "") {
+ std::stringstream ss;
+ ss << "property key " << prop << " has empty value. It is not allowed";
+ throw std::invalid_argument(ss.str());
+ }
+
+ layer->setProperty(static_cast<Layer::PropertyType>(i), value);
+ }
+
+ status = layer->setName(layer_name);
+ NN_RETURN_STATUS();
+
+ return ML_ERROR_NONE;
+}
+
+int ModelLoader::loadLayerConfigIni(dictionary *ini,
+ std::shared_ptr<Layer> &layer,
+ const std::string &layer_name) {
+ std::string layer_type_str =
+ iniparser_getstring(ini, (layer_name + ":Type").c_str(), unknown);
+ LayerType layer_type = (LayerType)parseType(layer_type_str, TOKEN_LAYER);
+
+ return loadLayerConfigIniCommon(ini, layer, layer_name, layer_type);
+}
+
+int ModelLoader::loadBackboneConfigIni(dictionary *ini,
+ const std::string &backbone_config,
+ NeuralNetwork &model,
+ const std::string &backbone_name) {
+ int status = ML_ERROR_NONE;
+ NeuralNetwork backbone;
+
+ bool trainable =
+ iniparser_getboolean(ini, (backbone_name + ":trainable").c_str(), false);
+
+ status = loadFromConfig(backbone_config, backbone, true);
+ NN_RETURN_STATUS();
+
+ auto graph = backbone.getGraph();
+ for (auto &layer : graph)
+ layer->setTrainable(trainable);
+
+ status = model.extendGraph(backbone.getGraph(), backbone_name);
+ NN_RETURN_STATUS();
+
+ return ML_ERROR_NONE;
+}
+
+int ModelLoader::loadBackboneConfigExternal(dictionary *ini,
+ const std::string &backbone_config,
+ std::shared_ptr<Layer> &layer,
+ const std::string &backbone_name) {
+ LayerType type = LayerType::LAYER_UNKNOWN;
+
+#if defined(ENABLE_NNSTREAMER_BACKBONE)
+ type = LayerType::LAYER_BACKBONE_NNSTREAMER;
+#endif
+
+ /** TfLite has higher priority */
+#if defined(ENABLE_TFLITE_BACKBONE)
+ if (fileTfLite(backbone_config))
+ type = LayerType::LAYER_BACKBONE_TFLITE;
+#endif
+
+ if (type == LayerType::LAYER_UNKNOWN)
+ return ML_ERROR_NOT_SUPPORTED;
+
+ int status = ML_ERROR_NONE;
+ status = loadLayerConfigIniCommon(ini, layer, backbone_name, type);
+ NN_RETURN_STATUS();
+
+ layer->setProperty(Layer::PropertyType::modelfile, backbone_config);
+ return status;
+}
+
+/**
+ * @brief load all of model and dataset from ini
+ */
+int ModelLoader::loadFromIni(std::string ini_file, NeuralNetwork &model,
+ bool bare_layers) {
+ int status = ML_ERROR_NONE;
+ int num_ini_sec = 0;
+ dictionary *ini;
+ const char model_str[] = "model";
+ unsigned int model_len = strlen(model_str);
+ const char dataset_str[] = "dataset";
+ unsigned int dataset_len = strlen(dataset_str);
+
+ if (ini_file.empty()) {
+ ml_loge("Error: Configuration File is not defined");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ if (!isFileExist(ini_file)) {
+ ml_loge("Cannot open model configuration file, filename : %s",
+ ini_file.c_str());
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ /** Parse ini file */
+ ini = iniparser_load(ini_file.c_str());
+ if (ini == NULL) {
+ ml_loge("Error: cannot parse file: %s\n", ini_file.c_str());
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ /** Get number of sections in the file */
+ num_ini_sec = iniparser_getnsec(ini);
+ if (num_ini_sec < 0) {
+ ml_loge("Error: invalid number of sections.");
+ status = ML_ERROR_INVALID_PARAMETER;
+ NN_INI_RETURN_STATUS();
+ }
+
+ if (!bare_layers) {
+ status = loadModelConfigIni(ini, model);
+ NN_INI_RETURN_STATUS();
+
+ status = loadDatasetConfigIni(ini, model);
+ NN_INI_RETURN_STATUS();
+ }
+
+ ml_logd("parsing ini started");
+ /** Get all the section names */
+ ml_logi("==========================parsing ini...");
+ ml_logi("invalid properties does not cause error, rather be ignored");
+ ml_logi("not-allowed property for the layer throws error");
+ ml_logi("valid property with invalid value throws error as well");
+ for (int idx = 0; idx < num_ini_sec; ++idx) {
+ std::string sec_name = iniparser_getsecname(ini, idx);
+ ml_logd("probing section name: %s", sec_name.c_str());
+
+ if (sec_name.empty()) {
+ ml_loge("Error: Unable to retrieve section names from ini.");
+ status = ML_ERROR_INVALID_PARAMETER;
+ NN_INI_RETURN_STATUS();
+ }
+
+ if (strncasecmp(model_str, sec_name.c_str(), model_len) == 0)
+ continue;
+
+ if (strncasecmp(dataset_str, sec_name.c_str(), dataset_len) == 0)
+ continue;
+
+ /** Parse all the layers defined as sections in order */
+ std::shared_ptr<Layer> layer;
+
+ /**
+ * If this section is a backbone, load backbone section from this
+ * @note The order of backbones in the ini file defines the order on the
+ * backbones in the model graph
+ */
+ const char *backbone =
+ iniparser_getstring(ini, (sec_name + ":Backbone").c_str(), unknown);
+ if (backbone == unknown) {
+ status = loadLayerConfigIni(ini, layer, sec_name);
+ } else if (fileIni(backbone)) {
+ status = loadBackboneConfigIni(ini, backbone, model, sec_name);
+ NN_INI_RETURN_STATUS();
+ continue;
+ } else {
+ status = loadBackboneConfigExternal(ini, backbone, layer, sec_name);
+ }
+ NN_INI_RETURN_STATUS();
+
+ status = model.addLayer(layer);
+ NN_INI_RETURN_STATUS();
+ }
+ ml_logd("parsing ini finished");
+
+ if (model.layers.empty()) {
+ ml_loge("there is no layer section in the ini file");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+
+ iniparser_freedict(ini);
+ return status;
+}
+
+/**
+ * @brief load all of model and dataset from given config file
+ */
+int ModelLoader::loadFromConfig(std::string config, NeuralNetwork &model) {
+ return loadFromConfig(config, model, false);
+}
+
+/**
+ * @brief load all of model and dataset from given config file
+ */
+int ModelLoader::loadFromConfig(std::string config, NeuralNetwork &model,
+ bool bare_layers) {
+ if (fileIni(config)) {
+ return loadFromIni(config, model, bare_layers);
+ }
+
+ return ML_ERROR_INVALID_PARAMETER;
+}
+
+bool ModelLoader::fileExt(const std::string &filename, const std::string &ext) {
+ size_t position = filename.find_last_of(".");
+ if (position == std::string::npos)
+ return false;
+
+ if (filename.substr(position + 1) == ext) {
+ return true;
+ }
+
+ return false;
+}
+
+bool ModelLoader::fileIni(const std::string &filename) {
+ return fileExt(filename, "ini");
+}
+
+bool ModelLoader::fileTfLite(const std::string &filename) {
+ return fileExt(filename, "tflite");
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file model_loader.h
+ * @date 5 August 2020
+ * @brief This is model loader class for the Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __MODEL_LOADER_H__
+#define __MODEL_LOADER_H__
+#ifdef __cplusplus
+
+#include <iniparser.h>
+#include <neuralnet.h>
+
+namespace nntrainer {
+
+/**
+ * @class ModelLoader
+ * @brief Model Loader class to load model from various config files
+ */
+class ModelLoader {
+public:
+ /**
+ * @brief Constructor of the model loader
+ */
+ ModelLoader() {}
+
+ /**
+ * @brief Destructor of the model loader
+ */
+ ~ModelLoader() {}
+
+ /**
+ * @brief load all of model and dataset from given config file
+ * @param[in] config config file path
+ * @param[in/out] model model to be loaded
+ */
+ int loadFromConfig(std::string config, NeuralNetwork &model);
+
+private:
+ /**
+ * @brief load all of model from given config file
+ * @param[in] config config file path
+ * @param[in/out] model model to be loaded
+ * @param[in] bare_layers load only the layers as backbone if enabled
+ */
+ int loadFromConfig(std::string config, NeuralNetwork &model,
+ bool bare_layers);
+
+ /**
+ * @brief load all of model and dataset from ini
+ * @param[in] ini_file config file path
+ * @param[in/out] model model to be loaded
+ */
+ int loadFromIni(std::string ini_file, NeuralNetwork &model, bool bare_layers);
+
+ /**
+ * @brief load dataset config from ini
+ * @param[in] ini dictionary containing the config
+ * @param[in] model model to be loaded
+ */
+ int loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model);
+
+ /**
+ * @brief load model config from ini
+ * @param[in] ini dictionary containing the config
+ * @param[in/out] model model to be loaded
+ */
+ int loadModelConfigIni(dictionary *ini, NeuralNetwork &model);
+
+ /**
+ * @brief load layer config from ini given the layer type
+ * @param[in] ini dictionary containing the config
+ * @param[in/out] layer layer to be loaded
+ * @param[in] layer_name name of the layer to be loaded
+ * @param[in] layer_type type of the layer to be loaded
+ */
+ int loadLayerConfigIniCommon(dictionary *ini, std::shared_ptr<Layer> &layer,
+ const std::string &layer_name,
+ LayerType layer_type);
+
+ /**
+ * @brief wrapper function to load layer config from ini
+ * @param[in] ini dictionary containing the config
+ * @param[in/out] layer layer to be loaded
+ * @param[in] layer_name name of the layer to be loaded
+ */
+ int loadLayerConfigIni(dictionary *ini, std::shared_ptr<Layer> &layer,
+ const std::string &layer_name);
+
+ /**
+ * @brief load backbone config from ini
+ * @param[in] ini dictionary containing the config
+ * @param[in] backbone_config config file containing the backbone config
+ * @param[in/out] model model to be added the backbone to
+ * @param[in] backbone_name name of the backbone to be loaded
+ */
+ int loadBackboneConfigIni(dictionary *ini, const std::string &backbone_config,
+ NeuralNetwork &model,
+ const std::string &backbone_name);
+
+ /**
+ * @brief wrapper function to load backbone config as layer from ini
+ * @param[in] ini dictionary containing the config
+ * @param[in] backbone_config config file containing the backbone config
+ * @param[in/out] model model to be added the backbone to
+ * @param[in] backbone_name name of the backbone to be loaded
+ * @note External implies that this backbone is dependent on external
+ * frameworks and this model will be treated as a blackbox by nntrainer.
+ * Training this backbone is dependent on the API exposed by the corresponding
+ * framework.
+ */
+ int loadBackboneConfigExternal(dictionary *ini,
+ const std::string &backbone_config,
+ std::shared_ptr<Layer> &layer,
+ const std::string &backbone_name);
+
+ /**
+ * @brief Check if the file extension is the given @a ext
+ * @param[in] filename full name of the file
+ * @param[in] ext extension to match with
+ * @retval true if @a ext, else false
+ */
+ static bool fileExt(const std::string &filename, const std::string &ext);
+
+ /**
+ * @brief Check if the file extension is ini
+ * @param[in] filename full name of the file
+ * @retval true if ini, else false
+ */
+ static bool fileIni(const std::string &filename);
+
+ /**
+ * @brief Check if the file extension is tflite
+ * @param[in] filename full name of the file
+ * @retval true if tflite, else false
+ */
+ static bool fileTfLite(const std::string &filename);
+
+ const char *unknown = "Unknown";
+};
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __MODEL_LOADER_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file neuralnet.cpp
+ * @date 04 December 2019
+ * @brief This is Neural Network Class
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <cmath>
+#include <fstream>
+#include <sstream>
+
+#include <databuffer_file.h>
+#include <databuffer_func.h>
+#include <model_loader.h>
+#include <neuralnet.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <unordered_set>
+#include <util_func.h>
+
+/**
+ * @brief Internal enum values for nntrainer to summarize model accuracy & loss
+ */
+#define ML_TRAIN_SUMMARY_MODEL_TRAIN_LOSS 101
+#define ML_TRAIN_SUMMARY_MODEL_VALID_LOSS 102
+#define ML_TRAIN_SUMMARY_MODEL_VALID_ACCURACY 103
+
+namespace nntrainer {
+
+int NeuralNetwork::loadFromConfig(std::string config) {
+ if (loadedFromConfig == true) {
+ ml_loge("cannnot do loadFromConfig twice");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ ModelLoader loader;
+ NeuralNetwork tempNet(*this);
+ int status = loader.loadFromConfig(config, tempNet);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+
+ tempNet.loadedFromConfig = true;
+ swap(tempNet, *this);
+
+ return ML_ERROR_NONE;
+}
+
+int NeuralNetwork::initLossLayer() {
+ int status = ML_ERROR_NONE;
+ LossType updated_loss_type = loss_type;
+
+ if (layers.empty()) {
+ status = ML_ERROR_INVALID_PARAMETER;
+ NN_RETURN_STATUS();
+ }
+
+ if (updated_loss_type == LossType::LOSS_ENTROPY) {
+ if (layers.back()->getType() != LayerType::LAYER_ACTIVATION) {
+ ml_loge("Error: Cross Entropy need last layer to have softmax or sigmoid "
+ "activation.");
+ return ML_ERROR_NOT_SUPPORTED;
+ }
+
+ NodeType act_layer = layers.back();
+ layers.pop_back();
+
+ switch (act_layer->getActivationType()) {
+ case ActivationType::ACT_SIGMOID:
+ updated_loss_type = LossType::LOSS_ENTROPY_SIGMOID;
+ break;
+ case ActivationType::ACT_SOFTMAX:
+ updated_loss_type = LossType::LOSS_ENTROPY_SOFTMAX;
+ break;
+ default:
+ ml_loge("Error: Cross Entropy not supported without softmax or sigmoid.");
+ return ML_ERROR_NOT_SUPPORTED;
+ }
+ }
+
+ std::shared_ptr<LossLayer> loss_layer = std::make_shared<LossLayer>();
+ ensureName(loss_layer);
+
+ loss_layer->setInputDimension(getOutputDimension());
+ status = loss_layer->initialize();
+ NN_RETURN_STATUS();
+
+ status = loss_layer->setLoss(updated_loss_type);
+ NN_RETURN_STATUS();
+
+ addLayer(std::static_pointer_cast<Layer>(loss_layer));
+ return status;
+}
+
+int NeuralNetwork::setProperty(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+ status = getKeyValue(values[i], key, value);
+ NN_RETURN_STATUS();
+
+ unsigned int type = parseNetProperty(key);
+
+ switch (static_cast<PropertyType>(type)) {
+ case PropertyType::loss: {
+ status = setFloat(loss, value);
+ NN_RETURN_STATUS();
+ } break;
+ case PropertyType::loss_type: {
+ loss_type = (LossType)parseType(value, TOKEN_LOSS);
+ } break;
+ default:
+ status = setTrainConfig({values[i]});
+ NN_RETURN_STATUS();
+ break;
+ }
+ }
+
+ return status;
+}
+
+int NeuralNetwork::setTrainConfig(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+ status = getKeyValue(values[i], key, value);
+ NN_RETURN_STATUS();
+
+ unsigned int type = parseNetProperty(key);
+
+ switch (static_cast<PropertyType>(type)) {
+ case PropertyType::epochs: {
+ status = setUint(epochs, value);
+ NN_RETURN_STATUS();
+ } break;
+ case PropertyType::save_path: {
+ save_path = value;
+ } break;
+ case PropertyType::continue_train: {
+ bool cont_train;
+ status = setBoolean(cont_train, value);
+ NN_RETURN_STATUS();
+ continue_train = cont_train;
+ opt->setProperty({values[i]});
+ } break;
+ case PropertyType::batch_size: {
+ status = setUint(batch_size, value);
+ NN_RETURN_STATUS();
+ /** TODO: increase buffer size if it is smaller than batch size.
+ * also if this is set with default batch size, then make it
+ * smaller/larger
+ */
+ } break;
+ default:
+ ml_loge("Error: Unknown Network Property Key");
+ status = ML_ERROR_INVALID_PARAMETER;
+ return status;
+ }
+ }
+
+ return status;
+}
+
+int NeuralNetwork::init() {
+ int status = ML_ERROR_NONE;
+ std::vector<TensorDim> previous_dim;
+
+ status = isInitializable();
+ NN_RETURN_STATUS();
+
+ ml_logd("initiating neural network, layer size: %d",
+ (unsigned int)layers.size());
+ /** Note: number of entries in layers will change. */
+ for (unsigned int i = 0; i < layers.size(); ++i) {
+ bool first = i == 0;
+ Layer &l = *layers[i];
+ ml_logd("layer name: %s", l.getName().c_str());
+
+ if (!first) {
+ if (layers[i - 1]->getType() == LayerType::LAYER_ACTIVATION &&
+ l.getType() == LayerType::LAYER_ACTIVATION) {
+ ml_loge("double activation is not allowed");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ if (l.getInputDimension().size()) {
+ l.setInputDimension(previous_dim);
+ } else if (previous_dim != l.getInputDimension()) {
+ ml_loge("Dimension mismatch between layers.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ }
+
+ status = layers[i]->initialize();
+ NN_RETURN_STATUS();
+
+ switch (l.getType()) {
+ case LayerType::LAYER_BN:
+ /// fallthrough intended
+ case LayerType::LAYER_CONV2D:
+ /// fallthrough intended
+ case LayerType::LAYER_FC:
+ status = l.setOptimizer(opt);
+ NN_RETURN_STATUS();
+ break;
+ default:
+ break;
+ }
+
+ if (l.getType() != LayerType::LAYER_ACTIVATION) {
+ status = realizeActivationType(l.getActivationType(), i);
+ NN_RETURN_STATUS();
+ }
+
+ if (l.getFlatten()) {
+ status = realizeFlattenType(i);
+ NN_RETURN_STATUS();
+ }
+
+ previous_dim = l.getOutputDimension();
+ }
+
+ /** Add the last layer as loss layer */
+ status = initLossLayer();
+ NN_RETURN_STATUS();
+
+ ml_logd("initialize successful, with layer size: %d", (int)layers.size());
+
+ for (auto l : layers)
+ ml_logd("layer name: %s", l->getName().c_str());
+
+ initialized = true;
+ setBatchSize(batch_size);
+ return status;
+}
+
+/**
+ * @brief free layers
+ */
+NeuralNetwork::~NeuralNetwork() {
+ layers.erase(layers.begin(), layers.end());
+
+ if (data_buffer) {
+ data_buffer->clear();
+ }
+}
+
+/**
+ * @brief forward propagation using layers object which has layer
+ */
+sharedConstTensors NeuralNetwork::forwarding(sharedConstTensors input) {
+ sharedConstTensors X = input;
+ /** Do not forward the loss layer, as label is not available */
+ for (unsigned int i = 0; i < layers.size() - 1; i++) {
+ X = layers[i]->forwarding(X);
+ }
+
+ return X;
+}
+
+/**
+ * @brief forward propagation using layers object which has layer
+ */
+sharedConstTensors NeuralNetwork::forwarding(sharedConstTensors input,
+ sharedConstTensors label) {
+ sharedConstTensors X;
+
+ if (input[0]->getDim().batch() > batch_size)
+ throw std::logic_error("Error: mismatch in batchsize for data and model.");
+
+ X = forwarding(input);
+ X = std::static_pointer_cast<LossLayer>(layers[layers.size() - 1])
+ ->forwarding(X, label);
+
+ return X;
+}
+
+/**
+ * @brief back propagation
+ * Call backwarding function of layer in reverse order
+ * No need to call at first Input Layer (No data to be updated)
+ */
+void NeuralNetwork::backwarding(sharedConstTensors input,
+ sharedConstTensors label, int iteration) {
+
+ if (layers.empty() || layers.back()->getType() != LayerType::LAYER_LOSS) {
+ throw std::invalid_argument("last layer is not loss layer");
+ }
+
+ forwarding(input, label);
+
+ sharedConstTensors output = label;
+ for (unsigned int i = layers.size() - 1; i > 0; i--)
+ output = layers[i]->backwarding(output, iteration);
+}
+
+float NeuralNetwork::getLoss() {
+ loss = 0.0f;
+ for (unsigned int i = 0; i < layers.size(); i++) {
+ loss += layers[i]->getLoss();
+ }
+ return loss;
+}
+
+void NeuralNetwork::setLoss(float l) { loss = l; }
+
+NeuralNetwork &NeuralNetwork::copy(NeuralNetwork &from) {
+ if (this != &from) {
+ batch_size = from.batch_size;
+ loss = from.loss;
+ opt = from.opt;
+
+ for (unsigned int i = 0; i < layers.size(); i++)
+ layers[i]->copy(from.layers[i]);
+ }
+ return *this;
+}
+
+/**
+ * @brief save model to file
+ * save Weight & Bias Data into file by calling save from layer
+ * save training parameters from the optimizer
+ */
+void NeuralNetwork::saveModel() {
+ std::ofstream model_file(save_path, std::ios::out | std::ios::binary);
+ for (unsigned int i = 0; i < layers.size(); i++)
+ layers[i]->save(model_file);
+ model_file.write((char *)&epoch_idx, sizeof(epoch_idx));
+ model_file.write((char *)&iter, sizeof(iter));
+ model_file.close();
+}
+
+/**
+ * @brief read model from file
+ * read Weight & Bias Data into file by calling save from layer
+ * read training parameters from the optimizer if continuing train
+ */
+void NeuralNetwork::readModel() {
+ if (!isFileExist(save_path))
+ return;
+ std::ifstream model_file(save_path, std::ios::in | std::ios::binary);
+ for (unsigned int i = 0; i < layers.size(); i++)
+ layers[i]->read(model_file);
+ if (continue_train) {
+ model_file.read((char *)&epoch_idx, sizeof(epoch_idx));
+ model_file.read((char *)&iter, sizeof(iter));
+ }
+ model_file.close();
+ ml_logi("read modelfile: %s", save_path.c_str());
+}
+
+void NeuralNetwork::setBatchSize(unsigned int batch) {
+ batch_size = batch;
+ for (auto const &layer : layers)
+ layer->setBatch(batch_size);
+
+ if (data_buffer && data_buffer->setBatchSize(batch_size) != ML_ERROR_NONE)
+ throw std::invalid_argument("Error setting batchsize for the dataset");
+}
+
+sharedConstTensors NeuralNetwork::inference(sharedConstTensors X) {
+ if (batch_size != X[0]->batch()) {
+ /**
+ * Note that inference resets batch_size of the previous train configuration
+ * Next train must set its batch_size if inference is run with this model.
+ */
+ setBatchSize(X[0]->batch());
+ }
+
+ sharedConstTensors out;
+ try {
+ out = forwarding(X);
+ /** Forward loss layer without label as well */
+ out = std::static_pointer_cast<LossLayer>(layers[layers.size() - 1])
+ ->forwarding(out);
+ } catch (...) {
+ ml_loge("Failed to inference Model");
+ return out;
+ }
+ return out;
+}
+
+int NeuralNetwork::train(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ if (data_buffer == nullptr) {
+ ml_loge("Cannot initialize the model without the data buffer.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ status = setTrainConfig(values);
+ NN_RETURN_STATUS();
+
+ /** set batch size just before training */
+ setBatchSize(batch_size);
+
+ /** Setup data buffer properties */
+ status = data_buffer->setClassNum(getOutputDimension()[0].width());
+ NN_RETURN_STATUS();
+
+ status = data_buffer->setFeatureSize(layers[0]->getInputDimension()[0]);
+ NN_RETURN_STATUS();
+
+ status = data_buffer->init();
+ NN_RETURN_STATUS();
+
+ return train_run();
+}
+
+/**
+ * @brief Run NeuralNetwork train with callback function by user
+ */
+int NeuralNetwork::train_run() {
+ int status = ML_ERROR_NONE;
+
+ for (epoch_idx = epoch_idx + 1; epoch_idx <= epochs; ++epoch_idx) {
+ training.loss = 0.0f;
+ status = data_buffer->run(nntrainer::BufferType::BUF_TRAIN);
+ if (status != ML_ERROR_NONE) {
+ data_buffer->clear(BufferType::BUF_TRAIN);
+ return status;
+ }
+
+ if (data_buffer->getValidation()[(int)nntrainer::BufferType::BUF_TEST]) {
+ status = data_buffer->run(nntrainer::BufferType::BUF_TEST);
+ if (status != ML_ERROR_NONE) {
+ data_buffer->clear(BufferType::BUF_TEST);
+ return status;
+ }
+ }
+
+ int count = 0;
+
+ sharedTensor in = MAKE_SHARED_TENSOR(getInputDimension()[0]);
+ sharedTensor label = MAKE_SHARED_TENSOR(getOutputDimension()[0]);
+
+ while (true) {
+ if (data_buffer->getDataFromBuffer(nntrainer::BufferType::BUF_TRAIN,
+ in->getData(), label->getData())) {
+ try {
+ backwarding({in}, {label}, iter++);
+ } catch (...) {
+ data_buffer->clear(nntrainer::BufferType::BUF_TRAIN);
+ ml_loge("Error: training error in #%d/%d.", epoch_idx, epochs);
+ std::rethrow_exception(std::current_exception());
+ }
+ std::cout << "#" << epoch_idx << "/" << epochs;
+ data_buffer->displayProgress(count++, nntrainer::BufferType::BUF_TRAIN,
+ getLoss());
+ training.loss += getLoss();
+ } else {
+ data_buffer->clear(nntrainer::BufferType::BUF_TRAIN);
+ break;
+ }
+ }
+
+ if (count == 0)
+ throw std::runtime_error("No training data");
+
+ training.loss /= count;
+ saveModel();
+
+ std::cout << "#" << epoch_idx << "/" << epochs
+ << " - Training Loss: " << training.loss;
+
+ if (data_buffer->getValidation()[(int)nntrainer::BufferType::BUF_VAL]) {
+ int right = 0;
+ validation.loss = 0.0f;
+ unsigned int tcases = 0;
+
+ status = data_buffer->run(nntrainer::BufferType::BUF_VAL);
+ if (status != ML_ERROR_NONE) {
+ data_buffer->clear(BufferType::BUF_VAL);
+ return status;
+ }
+
+ while (true) {
+ if (data_buffer->getDataFromBuffer(nntrainer::BufferType::BUF_VAL,
+ in->getData(), label->getData())) {
+ sharedConstTensors Y = forwarding({in}, {label});
+ auto model_out = Y[0]->argmax();
+ auto label_out = label->argmax();
+ for (unsigned int b = 0; b < batch_size; b++) {
+ if (model_out[b] == label_out[b])
+ right++;
+ }
+ validation.loss += getLoss();
+ tcases++;
+ } else {
+ data_buffer->clear(nntrainer::BufferType::BUF_VAL);
+ break;
+ }
+ }
+
+ if (tcases == 0) {
+ ml_loge("Error : 0 test cases");
+ status = ML_ERROR_INVALID_PARAMETER;
+ return status;
+ }
+ validation.loss /= (float)(tcases);
+ validation.accuracy = right / (float)(tcases * batch_size) * 100.0f;
+ std::cout << " >> [ Accuracy: " << validation.accuracy
+ << "% - Validation Loss : " << validation.loss << " ] ";
+ }
+ std::cout << std::endl;
+ }
+
+ return status;
+}
+
+int NeuralNetwork::isInitializable() {
+ if (layers.empty()) {
+ ml_loge("Layer is empty");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ Layer &l = *layers[0];
+
+ /** Dimension of first layer must be known */
+ if (l.getInputDimension().size() == 0) {
+ ml_loge("InputDimension of first layer is not set");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ /** First layer cannot be activation, batch normalization or loss */
+ switch (l.getType()) {
+ case LayerType::LAYER_ACTIVATION:
+ /// fallthrough intended
+ case LayerType::LAYER_BN:
+ /// fallthrough intended
+ case LayerType::LAYER_LOSS:
+ /// fallthrough intended
+ ml_loge("%s cannot be the first layer, type: %d", l.getName().c_str(),
+ static_cast<std::underlying_type<LayerType>::type>(l.getType()));
+ return ML_ERROR_INVALID_PARAMETER;
+ default:
+ /// normal case
+ break;
+ }
+
+ return ML_ERROR_NONE;
+}
+
+int NeuralNetwork::addLayer(NodeType layer) {
+ int status = ML_ERROR_NONE;
+
+ if (initialized) {
+ return ML_ERROR_NOT_SUPPORTED;
+ }
+
+ /** Ensure that the layer has a name and is unique */
+ ensureName(layer);
+
+ /** Validate the layer to be added */
+ status = layer->checkValidation();
+ if (status != ML_ERROR_NONE) {
+ ml_loge("layer(%s) validation failed.", layer->getName().c_str());
+ return status;
+ }
+
+ /** Insert the layer to the graph */
+ layers.push_back(layer);
+
+ return status;
+}
+
+int NeuralNetwork::extendGraph(GraphType graph, std::string prefix) {
+ if (initialized) {
+ return ML_ERROR_NOT_SUPPORTED;
+ }
+
+ /** Insert the layer to the graph */
+ for (auto layer : graph) {
+ /**
+ * Add prefix to the existing layer name,
+ * and ensure it is unique in this new graph
+ */
+ ensureName(layer, prefix, true);
+
+ layers.push_back(layer);
+ }
+
+ return ML_ERROR_NONE;
+}
+
+int NeuralNetwork::setOptimizer(
+ std::shared_ptr<ml::train::Optimizer> optimizer) {
+
+ if (optimizer->getType() == OptType::UNKNOWN)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ if (initialized) {
+ return ML_ERROR_NOT_SUPPORTED;
+ }
+
+ opt = std::static_pointer_cast<Optimizer>(optimizer);
+
+ return ML_ERROR_NONE;
+}
+
+int NeuralNetwork::setDataBuffer(std::shared_ptr<DataBuffer> data_buffer) {
+ this->data_buffer = data_buffer;
+
+ return ML_ERROR_NONE;
+}
+
+void NeuralNetwork::ensureName(NodeType layer, const std::string &prefix,
+ bool force_rename) {
+ std::string orig_name = layer->getName();
+ bool orig_name_empty = orig_name.empty();
+ if (!orig_name_empty && !force_rename &&
+ layer_names.end() == layer_names.find(orig_name))
+ return;
+
+ /** If just prefix with layer name makes it unique - directly set the name */
+ if (!orig_name_empty) {
+ std::string direct_name = prefix + orig_name;
+ if (layer_names.find(direct_name) == layer_names.end()) {
+ layer->setName(direct_name);
+ return;
+ }
+ }
+
+ std::set<std::string>::iterator iter;
+ std::string name;
+ if (orig_name_empty)
+ orig_name = layer->getBaseName();
+ std::string direct_name = prefix + orig_name;
+
+ do {
+ name = direct_name + std::to_string(def_name_count++);
+ iter = layer_names.find(name);
+ } while (iter != layer_names.end());
+
+ layer->setName(name);
+}
+
+int NeuralNetwork::getLayer(const char *name,
+ std::shared_ptr<ml::train::Layer> *layer) {
+ std::shared_ptr<Layer> layer_;
+ int ret = getLayer(name, &layer_);
+ if (ret == ML_ERROR_NONE)
+ *layer = layer_;
+ return ret;
+}
+
+int NeuralNetwork::getLayer(const char *name, NodeType *layer) {
+ int status = ML_ERROR_INVALID_PARAMETER;
+ std::string name_str(name);
+
+ for (auto iter = layers.begin(); iter != layers.end(); ++iter) {
+ if ((*iter)->getName() == name_str) {
+ *layer = *iter;
+ return ML_ERROR_NONE;
+ }
+ }
+
+ return status;
+}
+
+int NeuralNetwork::realizeActivationType(const ActivationType act) {
+ unsigned int position = layers.end() - layers.begin() - 1;
+ return realizeActivationType(act, position);
+}
+
+int NeuralNetwork::realizeActivationType(const ActivationType act,
+ const unsigned int position) {
+ if (act == ActivationType::ACT_NONE) {
+ /// ActivationType::ACT_NONE does not need realization
+ return ML_ERROR_NONE;
+ }
+
+ if (layers.empty()) {
+ ml_loge("layer is empty");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ Layer ¤t = *layers[position];
+ if (current.getType() == LayerType::LAYER_ACTIVATION) {
+ ml_loge("It is not allowed to realize ativation layer, possibly layer is "
+ "added right after activation");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ if (act == ActivationType::ACT_UNKNOWN) {
+ ml_loge("cannot realize unknown activation type");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ std::shared_ptr<ActivationLayer> act_layer =
+ std::make_shared<ActivationLayer>();
+ ensureName(act_layer, current.getName());
+ act_layer->setActivation(act);
+
+ layers.insert(layers.begin() + position + 1, act_layer);
+ return ML_ERROR_NONE;
+}
+
+int NeuralNetwork::realizeFlattenType(const unsigned int position) {
+ if (layers.empty()) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ Layer ¤t = *layers[position];
+ if (current.getType() == LayerType::LAYER_FLATTEN) {
+ ml_loge(
+ "It is not allowed to realize flatten layer, possibly flatten layer is "
+ "added right after flatten");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ std::shared_ptr<FlattenLayer> flatten_layer =
+ std::make_shared<FlattenLayer>();
+
+ ensureName(flatten_layer, current.getName());
+ layers.insert(layers.begin() + position + 1, flatten_layer);
+ return ML_ERROR_NONE;
+}
+
+int NeuralNetwork::realizeFlattenType() {
+ unsigned int position = layers.end() - layers.begin() - 1;
+ return realizeFlattenType(position);
+}
+
+/**
+ * @brief Set loss type for the neural network.
+ */
+int NeuralNetwork::setLoss(LossType loss_type) {
+ if (loss_type == LossType::LOSS_UNKNOWN)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ this->loss_type = loss_type;
+ return ML_ERROR_NONE;
+}
+
+void NeuralNetwork::printMetrics(std::ostream &out, unsigned int flags) {
+ switch (flags) {
+ case ML_TRAIN_SUMMARY_MODEL_TRAIN_LOSS:
+ out << training.loss << std::endl;
+ break;
+
+ case ML_TRAIN_SUMMARY_MODEL_VALID_LOSS:
+ out << validation.loss << std::endl;
+ break;
+
+ case ML_TRAIN_SUMMARY_MODEL_VALID_ACCURACY:
+ out << validation.accuracy << std::endl;
+ break;
+
+ default:
+ break;
+ }
+}
+
+void NeuralNetwork::printPreset(std::ostream &out, unsigned int preset) {
+ /** print neuralnet metrics */
+ printMetrics(out, preset);
+ if (preset > ML_TRAIN_SUMMARY_TENSOR)
+ return;
+
+ Layer::PrintPreset layer_preset = Layer::PrintPreset::PRINT_NONE;
+
+ ///@todo match flags with preset
+ unsigned int flags = PRINT_INST_INFO | PRINT_GRAPH_INFO | PRINT_PROP |
+ PRINT_OPTIMIZER | PRINT_METRIC;
+
+ switch (preset) {
+ case ML_TRAIN_SUMMARY_TENSOR:
+ layer_preset = Layer::PrintPreset::PRINT_ALL;
+ break;
+ case ML_TRAIN_SUMMARY_LAYER:
+ layer_preset = initialized ? Layer::PrintPreset::PRINT_SUMMARY
+ : Layer::PrintPreset::PRINT_SUMMARY_META;
+ break;
+ case ML_TRAIN_SUMMARY_MODEL:
+ break;
+ default:
+ throw std::invalid_argument("given verbosity is invalid");
+ }
+
+ print(out, flags, layer_preset);
+}
+
+void NeuralNetwork::print(std::ostream &out, unsigned int flags,
+ Layer::PrintPreset layerPrintPreset) {
+ if (flags & PRINT_INST_INFO) {
+ out << "===================";
+ printInstance(out, this);
+ }
+
+ if (flags & PRINT_GRAPH_INFO) {
+ out << "graph contains " << layers.size() << " operation nodes\n";
+ /// @todo print graph info
+ }
+
+ if (flags & PRINT_PROP) {
+ /// @todo print neuralnet property
+ /// @todo print mode (if it is eval or training)
+ }
+
+ if (flags & PRINT_OPTIMIZER) {
+ /// @todo print optimizer (with print optimizer prop)
+ }
+
+ if (flags & PRINT_METRIC) {
+ /// @todo print metric (currently it is done at printPreset as a workaround)
+ /// @todo print loss function when it is not initialized. (if it is
+ /// initialized, loss layer will be printed)
+ }
+
+ if (layers.empty()) {
+ out << "model is empty!" << std::endl;
+ return;
+ }
+
+ /** print layer properties */
+ for (auto &layer : layers)
+ layer->printPreset(out, layerPrintPreset);
+
+ /// @todo Add status to check neuralnet has been run. #290
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file neuralnet.h
+ * @date 04 December 2019
+ * @brief This is Neural Network Class
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+#ifndef __NEURALNET_H__
+#define __NEURALNET_H__
+#ifdef __cplusplus
+
+#include <memory>
+#include <vector>
+
+#include <activation_layer.h>
+#include <bn_layer.h>
+#include <conv2d_layer.h>
+#include <databuffer.h>
+#include <fc_layer.h>
+#include <flatten_layer.h>
+#include <input_layer.h>
+#include <layer_internal.h>
+#include <loss_layer.h>
+#include <ml-api-common.h>
+#include <optimizer_internal.h>
+#include <pooling2d_layer.h>
+#include <tensor.h>
+
+#include <model.h>
+#include <nntrainer-api-common.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Enumeration of Network Type
+ */
+using NetType = ml::train::ModelType;
+
+/**
+ * @brief Statistics from running or training a model
+ */
+typedef struct RunStats_ {
+ float accuracy; /** accuracy of the model */
+ float loss; /** loss of the model */
+
+ RunStats_() : accuracy(0), loss(0) {}
+} RunStats;
+
+/**
+ * @class NeuralNetwork Class
+ * @brief NeuralNetwork Class which has Network Configuration & Layers
+ */
+class NeuralNetwork : public ml::train::Model {
+ friend class ModelLoader; /** access private members of ModelLoader */
+
+public:
+ using NodeType = std::shared_ptr<Layer>; /** Type of a Node */
+ using GraphType = std::vector<NodeType>; /** actual graph type */
+ using FlatGraphType =
+ std::vector<NodeType>; /** topological sorted, iterable 1-D list of nodes */
+
+ /**
+ * @brief Constructor of NeuralNetwork Class
+ */
+ NeuralNetwork() :
+ batch_size(1),
+ epochs(1),
+ epoch_idx(0),
+ iter(0),
+ loss(0.0f),
+ loss_type(LossType::LOSS_UNKNOWN),
+ weight_initializer(WeightInitializer::WEIGHT_UNKNOWN),
+ net_type(NetType::UNKNOWN),
+ data_buffer(nullptr),
+ continue_train(false),
+ initialized(false),
+ def_name_count(0),
+ loadedFromConfig(false) {}
+
+ /**
+ * @brief Destructor of NeuralNetwork Class
+ */
+ ~NeuralNetwork();
+
+ /**
+ * @brief Get Loss from the previous ran batch of data
+ * @retval loss value
+ */
+ float getLoss();
+
+ /**
+ * @brief Get Loss from the previous epoch of training data
+ * @retval loss value
+ */
+ float getTrainingLoss() { return training.loss; }
+
+ /**
+ * @brief Get Loss from the previous epoch of validation data
+ * @retval loss value
+ */
+ float getValidationLoss() { return validation.loss; }
+
+ /**
+ * @brief Get Learning rate
+ * @retval Learning rate
+ */
+ float getLearningRate() { return opt->getLearningRate(); };
+
+ /**
+ * @brief Create and load the Network with ini configuration file.
+ * @param[in] config config file path
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int loadFromConfig(std::string config);
+
+ /**
+ * @brief set Property of Network
+ * @param[in] values values of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(std::vector<std::string> values);
+
+ /**
+ * @brief Initialize Network. This should be called after set all
+ * hyperparameters.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int init();
+
+ /**
+ * @brief Forward Propagation of the neural network
+ * @param[in] input List of Input Tensors taken by the neural network
+ * @retval List of Output Tensors
+ */
+ sharedConstTensors forwarding(sharedConstTensors input);
+
+ /**
+ * @brief Forward Propagation of the neural network
+ * @param[in] input List of Input Tensors taken by the neural network
+ * @param[in] label List of Label Tensors for the model
+ * @retval List of Output Tensors
+ */
+ sharedConstTensors forwarding(sharedConstTensors input,
+ sharedConstTensors label);
+
+ /**
+ * @brief Backward Propagation of the neural network
+ * @param[in] input List of Input Tensors taken by the neural network
+ * @param[in] label List of Label Tensors for the model
+ * @param[in] iteration Iteration Number for the optimizer
+ */
+ void backwarding(sharedConstTensors input, sharedConstTensors label,
+ int iteration);
+
+ /**
+ * @brief save model and training parameters into file
+ */
+ void saveModel();
+
+ /**
+ * @brief read model and training parameters from file
+ */
+ void readModel();
+
+ /**
+ * @brief get Epochs
+ * @retval epochs
+ */
+ unsigned int getEpochs() { return epochs; };
+
+ /**
+ * @brief Copy Neural Network
+ * @param[in] from NeuralNetwork Object to copy
+ * @retval NeuralNewtork Object copyed
+ */
+ NeuralNetwork ©(NeuralNetwork &from);
+
+ /**
+ * @brief Run NeuralNetwork train
+ * @param[in] values hyper parameters
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int train(std::vector<std::string> values = {});
+
+ /**
+ * @brief Run NeuralNetwork inference
+ * @param[in] X input tensor
+ * @retval shared_ptr<const Tensor>
+ */
+ sharedConstTensors inference(sharedConstTensors X);
+
+ /**
+ * @brief Run NeuralNetwork train with callback function by user
+ * @param[in] dataset set the dataset
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setDataset(std::shared_ptr<ml::train::Dataset> dataset) {
+ return setDataBuffer(std::static_pointer_cast<DataBuffer>(dataset));
+ }
+
+ /**
+ * @brief Run NeuralNetwork train with callback function by user
+ * @param[in] databuffer set the databuffer
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setDataBuffer(std::shared_ptr<DataBuffer> data_buffer);
+
+ /**
+ * @brief add layer into neural network model
+ * @param[in] layer layer to add
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int addLayer(std::shared_ptr<ml::train::Layer> layer) {
+ return addLayer(std::static_pointer_cast<Layer>(layer));
+ }
+
+ /**
+ * @brief add layer into neural network model
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int addLayer(NodeType layer);
+
+ /**
+ * @brief join passed graph into the existing graph model
+ * @param[in] graph graph to be added/to extend
+ * @param[in] prefix prefix added to names of layers from this graph
+ * @note It is assumed that this model is valid by itself
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int extendGraph(GraphType graph, std::string prefix = "");
+
+ /**
+ * @brief set optimizer for the neural network model
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setOptimizer(std::shared_ptr<ml::train::Optimizer> optimizer);
+
+ /*
+ * @brief get layer by name from neural network model
+ * @param[in] name name of the layer to get
+ * @param[out] layer shared_ptr to hold the layer to get
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int getLayer(const char *name, std::shared_ptr<ml::train::Layer> *layer);
+
+ /*
+ * @brief get layer by name from neural network model
+ * @param[in] name name of the layer to get
+ * @param[out] layer shared_ptr to hold the layer to get
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int getLayer(const char *name, NodeType *layer);
+
+ /*
+ * @brief get input dimension of neural network
+ * @retval std::vector<TensorDim> input dimension
+ */
+ std::vector<TensorDim> getInputDimension() {
+ return layers[0]->getInputDimension();
+ }
+
+ /*
+ * @brief get output dimension of neural network
+ * @retval std::vector<TensorDim> output dimension
+ */
+ std::vector<TensorDim> getOutputDimension() {
+ return layers.back()->getOutputDimension();
+ }
+
+ /**
+ * @brief get FlatGraph of current graph
+ * @note flat graph contains pointer to the actual nodes, which is not deeply
+ * copied.
+ * @retval flatGraph of the current graph
+ */
+ FlatGraphType getFlatGraph() { return layers; }
+
+ /**
+ * @brief get current graph from the model
+ * @note graph contains pointer to the actual nodes, which is not deeply
+ * copied.
+ * @retval current graph
+ */
+ GraphType getGraph() { return layers; }
+
+ /**
+ * @brief Set loss type for the neural network.
+ * @param[in] loss Type of the loss.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setLoss(LossType loss);
+
+ /**
+ * @brief Print Option when printing model info. The function delegates to the
+ * `print`
+ * @param out std::ostream to print
+ * @param preset preset from `ml_train_summary_type_e`
+ */
+ void printPreset(std::ostream &out, unsigned int preset);
+
+private:
+ /**
+ * @brief Print Options when printing layer info
+ */
+ typedef enum {
+ // clang-format off
+ PRINT_INST_INFO = (1 << 0), /**< Option to print type & instance address info */
+ PRINT_GRAPH_INFO = (1 << 1), /**< Option to print graph topology info */
+ PRINT_PROP = (1 << 2), /**< Option to print properties */
+ PRINT_OPTIMIZER = (1 << 3), /**< Option to print optimizer */
+ PRINT_METRIC = (1 << 4), /**< Option to print if current network is set to training */
+ // clang-format on
+ } PrintOption;
+
+ unsigned int batch_size; /**< batch size */
+
+ unsigned int epochs; /**< Maximum Epochs */
+
+ unsigned int epoch_idx; /**< Number of epoch_idx */
+
+ unsigned int iter; /**< iterations trained */
+
+ float loss; /**< loss */
+
+ LossType loss_type; /**< Loss Function type */
+
+ WeightInitializer weight_initializer; /**< Weight Initialization type */
+
+ std::string save_path; /**< Model path to save / read */
+
+ std::shared_ptr<Optimizer> opt; /**< Optimizer; this gets copied into each
+ layer, do not use this directly */
+
+ NetType net_type; /**< Network Type */
+
+ GraphType layers; /**< vector for store layer pointers */
+
+ std::shared_ptr<DataBuffer> data_buffer; /**< Data Buffer to get Input */
+
+ bool continue_train; /**< Continue train from the previous state of optimizer
+ and iterations */
+
+ bool initialized; /**< Network is initialized */
+
+ std::set<std::string>
+ layer_names; /**< Set containing all the names of layers in the model */
+
+ int def_name_count; /**< Count assigned to layer names declared by default */
+
+ bool loadedFromConfig; /**< Check if config is loaded to prevent load twice */
+
+ RunStats validation; /** validation statistics of the model */
+ RunStats training; /** training statistics of the model */
+ RunStats testing; /** testing statistics of the model */
+
+ /**
+ * @brief print function for neuralnet
+ * @param[in] out outstream
+ * @param[in] flags bit combination of Neuralnet::PrintOption
+ * @param[in] Layer::PrintPreset print preset when to print layer properties
+ */
+ void print(
+ std::ostream &out, unsigned int flags = 0,
+ Layer::PrintPreset layerPrintPreset = Layer::PrintPreset::PRINT_SUMMARY);
+
+ /**
+ * @brief Sets up and initialize the loss layer
+ */
+ int initLossLayer();
+
+ /**
+ * @brief Set Loss
+ * @param[in] l loss value
+ */
+ void setLoss(float l);
+
+ /**
+ * @brief Run NeuralNetwork train
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int train_run();
+
+ /**
+ * @brief check neural network is ready to init.
+ * @retval #ML_ERROR_NONE neuralnetwork is ready to init
+ * @retval #ML_ERROR_INVALID_PARAMETER not ready to init.
+ */
+ int isInitializable();
+
+ /**
+ * @brief Realize act type to layer and insert it to layers
+ * @param[in] ActivationType act Activation Type
+ * @param[in] int Position position to insert activation layer.
+ * @note layer is inserted at position
+ */
+ int realizeActivationType(const ActivationType act,
+ const unsigned int position);
+
+ /**
+ * @copydoc int realizeActivationType(ActivationType act, unsigned int
+ * &position);
+ * @note layer is inserted at the back of layers
+ */
+ int realizeActivationType(const ActivationType act);
+
+ /**
+ * @brief Realize flatten type to layer and insert it to layers
+ * @param[in] int Position position to insert the layer.
+ * @note layer is inserted at position
+ */
+ int realizeFlattenType(const unsigned int position);
+
+ /**
+ * @copydoc int realizeActivationType(ActivationType act, unsigned int
+ * &position);
+ * @note layer is inserted at the back of layers
+ */
+ int realizeFlattenType();
+
+ /**
+ * @brief Ensure that layer has a name
+ */
+ void ensureName(NodeType layer, const std::string &prefix = "",
+ bool force_rename = false);
+
+ /**
+ * @brief Swap function for the class
+ */
+ friend void swap(NeuralNetwork &lhs, NeuralNetwork &rhs) {
+ using std::swap;
+
+ swap(lhs.batch_size, rhs.batch_size);
+ swap(lhs.epochs, rhs.epochs);
+ swap(lhs.epoch_idx, rhs.epoch_idx);
+ swap(lhs.iter, rhs.iter);
+ swap(lhs.loss, rhs.loss);
+ swap(lhs.loss_type, rhs.loss_type);
+ swap(lhs.weight_initializer, rhs.weight_initializer);
+ swap(lhs.save_path, rhs.save_path);
+ swap(lhs.opt, rhs.opt);
+ swap(lhs.net_type, rhs.net_type);
+ swap(lhs.layers, rhs.layers);
+ swap(lhs.data_buffer, rhs.data_buffer);
+ swap(lhs.continue_train, rhs.continue_train);
+ swap(lhs.initialized, rhs.initialized);
+ swap(lhs.layer_names, rhs.layer_names);
+ swap(lhs.def_name_count, rhs.def_name_count);
+ swap(lhs.loadedFromConfig, rhs.loadedFromConfig);
+ }
+
+ /**
+ * @brief set Property/Configuration of Network for training after the
+ * network has been initialized
+ * @param[in] values values of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setTrainConfig(std::vector<std::string> values);
+
+ /**
+ * @brief Update batch size of the model as well as its layers/dataset
+ */
+ void setBatchSize(unsigned int batch_size);
+
+ /**
+ * @brief print metrics function for neuralnet
+ * @param[in] out outstream
+ * @param[in] flags verbosity from ml_train_summary_type_e
+ */
+ void printMetrics(std::ostream &out, unsigned int flags = 0);
+};
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __NEURALNET_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file nntrainer_error.h
+ * @date 03 April 2020
+ * @brief NNTrainer Error Codes
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+
+#ifndef __NNTRAINER_ERROR_H__
+#define __NNTRAINER_ERROR_H__
+
+#include <ml-api-common.h>
+#if defined(__TIZEN__)
+#include <tizen_error.h>
+#define ML_ERROR_BAD_ADDRESS TIZEN_ERROR_BAD_ADDRESS
+#define ML_ERROR_RESULT_OUT_OF_RANGE TIZEN_ERROR_RESULT_OUT_OF_RANGE
+#else
+#include <cerrno>
+#define ML_ERROR_BAD_ADDRESS (-EFAULT)
+#define ML_ERROR_RESULT_OUT_OF_RANGE (-ERANGE)
+#endif
+
+#include <stdexcept>
+namespace nntrainer {
+
+/// @note underscore_case is used for ::exception to keep in accordance with
+/// std::exception
+namespace exception {
+
+/**
+ * @brief derived class of invalid argument to represent specific functionality
+ * not supported
+ * @note this could be either intended or not yet implemented
+ */
+struct not_supported : public std::invalid_argument {
+ using invalid_argument::invalid_argument;
+};
+
+} // namespace exception
+
+} // namespace nntrainer
+
+#endif /* __NNTRAINER_ERROR_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file nntrainer_log.h
+ * @date 06 April 2020
+ * @brief NNTrainer Logger.
+ * Log Util for NNTrainer
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+
+#ifndef __NNTRAINER_LOG_H__
+#define __NNTRAINER_LOG_H__
+
+#define TAG_NAME "nntrainer"
+
+#if defined(__TIZEN__)
+#include <dlog.h>
+
+#define ml_logi(...) dlog_print(DLOG_INFO, TAG_NAME, __VA_ARGS__)
+
+#define ml_logw(...) dlog_print(DLOG_WARN, TAG_NAME, __VA_ARGS__)
+
+#define ml_loge(...) dlog_print(DLOG_ERROR, TAG_NAME, __VA_ARGS__)
+
+#define ml_logd(...) dlog_print(DLOG_DEBUG, TAG_NAME, __VA_ARGS__)
+
+#elif defined(__ANDROID__)
+#include <android/log.h>
+
+#define ml_logi(...) \
+ __android_log_print(ANDROID_LOG_INFO, TAG_NAME, __VA_ARGS__)
+
+#define ml_logw(...) \
+ __android_log_print(ANDROID_LOG_WARN, TAG_NAME, __VA_ARGS__)
+
+#define ml_loge(...) \
+ __android_log_print(ANDROID_LOG_ERROR, TAG_NAME, __VA_ARGS__)
+
+#define ml_logd(...) \
+ __android_log_print(ANDROID_LOG_DEBUG, TAG_NAME, __VA_ARGS__)
+
+#else /* Linux distro */
+#include <nntrainer_logger.h>
+
+#if !defined(ml_logi)
+#define ml_logi(format, args...) \
+ __nntrainer_log_print(NNTRAINER_LOG_INFO, "(%s:%s:%d) " format, __FILE__, \
+ __func__, __LINE__, ##args)
+#endif
+
+#if !defined(ml_logw)
+#define ml_logw(format, args...) \
+ __nntrainer_log_print(NNTRAINER_LOG_WARN, "(%s:%s:%d) " format, __FILE__, \
+ __func__, __LINE__, ##args)
+#endif
+
+#if !defined(ml_loge)
+#define ml_loge(format, args...) \
+ __nntrainer_log_print(NNTRAINER_LOG_ERROR, "(%s:%s:%d) " format, __FILE__, \
+ __func__, __LINE__, ##args)
+#endif
+
+#if !defined(ml_logd)
+#define ml_logd(format, args...) \
+ __nntrainer_log_print(NNTRAINER_LOG_DEBUG, "(%s:%s:%d) " format, __FILE__, \
+ __func__, __LINE__, ##args)
+#endif
+
+#endif
+
+#endif /* __NNTRAINER_LOG_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file nntrainer_logger.cpp
+ * @date 02 April 2020
+ * @brief NNTrainer Logger
+ * This allows to logging nntrainer logs.
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+
+#include <cstring>
+#include <ctime>
+#include <iomanip>
+#include <mutex>
+#include <nntrainer_logger.h>
+#include <sstream>
+#include <stdarg.h>
+#include <stdexcept>
+
+namespace nntrainer {
+
+/**
+ * @brief logfile name
+ */
+const char *const Logger::logfile_name = "log_nntrainer_";
+
+/**
+ * @brief instance for single logger
+ */
+Logger *Logger::ainstance = nullptr;
+
+/**
+ * @brief mutex for lock
+ */
+std::mutex Logger::smutex;
+
+Logger &Logger::instance() {
+ static Cleanup cleanup;
+
+ std::lock_guard<std::mutex> guard(smutex);
+ if (ainstance == nullptr)
+ ainstance = new Logger();
+ return *ainstance;
+}
+
+Logger::Cleanup::~Cleanup() {
+ std::lock_guard<std::mutex> guard(Logger::smutex);
+ delete Logger::ainstance;
+ Logger::ainstance = nullptr;
+}
+
+Logger::~Logger() { outputstream.close(); }
+
+Logger::Logger() {
+ struct tm lt;
+ time_t t = time(0);
+ struct tm *now = localtime_r(&t, <);
+ std::stringstream ss;
+ ss << logfile_name << std::dec << (now->tm_year + 1900) << std::setfill('0')
+ << std::setw(2) << (now->tm_mon + 1) << std::setfill('0') << std::setw(2)
+ << now->tm_mday << std::setfill('0') << std::setw(2) << now->tm_hour
+ << std::setfill('0') << std::setw(2) << now->tm_min << std::setfill('0')
+ << std::setw(2) << now->tm_sec << ".out";
+ outputstream.open(ss.str(), std::ios_base::app);
+ if (!outputstream.good()) {
+ throw std::runtime_error("Unable to initialize the Logger!");
+ }
+}
+
+void Logger::log(const std::string &message,
+ const nntrainer_loglevel loglevel) {
+ std::lock_guard<std::mutex> guard(smutex);
+ time_t t = time(0);
+ struct tm lt;
+ struct tm *now = localtime_r(&t, <);
+ std::stringstream ss;
+ switch (loglevel) {
+ case NNTRAINER_LOG_INFO:
+ ss << "[NNTRAINER INFO ";
+ break;
+ case NNTRAINER_LOG_WARN:
+ ss << "[NNTRAINER WARN ";
+ break;
+ case NNTRAINER_LOG_ERROR:
+ ss << "[NNTRAINER ERROR ";
+ break;
+ case NNTRAINER_LOG_DEBUG:
+ ss << "[NNTRAINER DEBUG ";
+ break;
+ default:
+ break;
+ }
+
+ ss << std::dec << (now->tm_year + 1900) << '-' << std::setfill('0')
+ << std::setw(2) << (now->tm_mon + 1) << '-' << std::setfill('0')
+ << std::setw(2) << now->tm_mday << ' ' << std::setfill('0') << std::setw(2)
+ << now->tm_hour << ':' << std::setfill('0') << std::setw(2) << now->tm_min
+ << ':' << std::setfill('0') << std::setw(2) << now->tm_sec << ']';
+
+ outputstream << ss.str() << " " << message << std::endl;
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+void __nntrainer_log_print(nntrainer_loglevel loglevel,
+ const std::string format_str, ...) {
+ int final_n, n = ((int)format_str.size()) * 2;
+ std::unique_ptr<char[]> formatted;
+ va_list ap;
+ while (1) {
+ formatted.reset(new char[n]);
+ std::strncpy(&formatted[0], format_str.c_str(), format_str.size());
+ va_start(ap, format_str);
+ final_n = vsnprintf(&formatted[0], n, format_str.c_str(), ap);
+ va_end(ap);
+ if (final_n < 0 || final_n >= n)
+ n += abs(final_n - n + 1);
+ else
+ break;
+ }
+
+ std::string ss = std::string(formatted.get());
+
+#if defined(__LOGGING__)
+ Logger::instance().log(ss, loglevel);
+#else
+
+ switch (loglevel) {
+ case NNTRAINER_LOG_ERROR:
+ std::cerr << ss << std::endl;
+ break;
+ case NNTRAINER_LOG_INFO:
+ case NNTRAINER_LOG_WARN:
+ case NNTRAINER_LOG_DEBUG:
+ std::cout << ss << std::endl;
+ default:
+ break;
+ }
+
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * @file nntrainer_logger.h
+ * @date 02 April 2020
+ * @brief NNTrainer Logger
+ * This allows to logging nntrainer logs.
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ */
+#ifndef __NNTRAINER_LOGGER_H___
+#define __NNTRAINER_LOGGER_H___
+#ifdef __cplusplus
+
+#include <fstream>
+#include <mutex>
+#include <string>
+
+/**
+ * @brief Log Level of NNtrainer
+ * 0. informations
+ * 1. warnings
+ * 2. errors
+ * 3. debugging informations
+ */
+typedef enum {
+ NNTRAINER_LOG_DEBUG = 0,
+ NNTRAINER_LOG_INFO,
+ NNTRAINER_LOG_WARN,
+ NNTRAINER_LOG_ERROR
+} nntrainer_loglevel;
+
+namespace nntrainer {
+
+/**
+ * @class NNTrainer Logger Class
+ * @brief Class for Logging. This is alternatives when there is no logging
+ * system. For the tizen, we are going to use dlog and it is android_log for
+ * android.
+ */
+class Logger {
+public:
+ /**
+ * @brief Logging Instance Function. Get a lock and create Logger if it
+ * is null;
+ */
+ static Logger &instance();
+
+ /**
+ * @brief Logging member function for logging messages.
+ */
+ void log(const std::string &message,
+ const nntrainer_loglevel loglevel = NNTRAINER_LOG_INFO);
+
+protected:
+ /**
+ * @brief Logging instance
+ */
+ static Logger *ainstance;
+ /**
+ * @brief Log file name
+ */
+ static const char *const logfile_name;
+
+ /**
+ * @brief output stream
+ */
+ std::ofstream outputstream;
+
+ /**
+ * @class Class to make sure the single logger
+ * @brief sure the single logger
+ */
+ friend class Cleanup;
+ class Cleanup {
+ public:
+ ~Cleanup();
+ };
+
+private:
+ /**
+ * @brief Constructor
+ */
+ Logger();
+ /**
+ * @brief Destructor
+ */
+ virtual ~Logger();
+ Logger(const Logger &);
+ Logger &operator=(const Logger &);
+ static std::mutex smutex;
+};
+} /* namespace nntrainer */
+
+extern "C" {
+
+#endif /* __cplusplus */
+
+/**
+ * @brief Interface function for C
+ */
+void __nntrainer_log_print(nntrainer_loglevel loglevel,
+ const std::string format, ...);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* __NNTRAINER_LOGGER_H___ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file adam.cpp
+ * @date 6 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the Adam optimizer.
+ */
+
+#include <cmath>
+#include <fstream>
+
+#include <adam.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int Adam::initialize(std::shared_ptr<Weight> weight_list,
+ unsigned int num_weights, bool set_tensor) {
+ int status = ML_ERROR_NONE;
+ weight_mv.clear();
+
+ if (set_tensor) {
+ for (unsigned int i = 0; i < num_weights; ++i) {
+ Weight &w = weight_list.get()[i];
+
+ // TODO: only trainable weights must be sent to optimizer
+ if (!w.getTrainable())
+ continue;
+
+ Tensor m = Tensor(w.getDim());
+ m.setZero();
+ Tensor v = Tensor(w.getDim());
+ v.setZero();
+ std::pair<Tensor, Tensor> p =
+ std::pair<Tensor, Tensor>(std::move(m), std::move(v));
+ weight_mv.push_back(std::move(p));
+ }
+ }
+ return status;
+}
+
+double Adam::getLearningRate(int iteration) {
+ double ll = Optimizer::getLearningRate(iteration);
+
+ std::function<float(double)> biasCorrection = [&](float f) {
+ return 1.0f - pow(f, iteration + 1);
+ };
+
+ ll *= sqrt(biasCorrection(beta2)) / biasCorrection(beta1);
+
+ return ll;
+}
+
+void Adam::apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ int iteration) {
+
+ Tensor &x = weight.getVariableRef();
+ const Tensor &x_grad = weight.getGradientRef();
+
+ // This is implementation of adam from original paper.
+ // This is not deleted intentionally.
+ // float biasCorrection1 = 1 - pow(beta1, iteration + 1);
+ // float biasCorrection2 = 1 - pow(beta2, iteration + 1);
+ // Tensor &wm = weight_mv[idx].first;
+ // Tensor &wv = weight_mv[idx].second;
+
+ // wm.multiply_i(beta1);
+ // wm.add_i(x_grad, 1.0f - beta1);
+
+ // wv.multiply_i(beta2);
+ // wv.add_i(x_grad.multiply(x_grad), 1.0f - beta2);
+
+ // Tensor denom = wv.apply(sqrtFloat)
+ // .divide(sqrtFloat(biasCorrection2))
+ // .add(epsilon);
+ // x.add_i(wm.divide(denom), -ll / biasCorrection1);
+
+ std::function<double(double)> sqrtEps = [&](double f) {
+ return sqrtDouble(f) + this->epsilon;
+ };
+
+ Tensor &wm = weight_mv[tensor_idx].first;
+ Tensor &wv = weight_mv[tensor_idx].second;
+
+ wm.multiply_i(beta1);
+ wm.add_i(x_grad, 1.0f - beta1);
+
+ wv.multiply_i(beta2);
+ wv.add_i(x_grad.multiply(x_grad), 1.0f - beta2);
+
+ x.add_i(wm.divide(wv.apply(sqrtEps)), -updated_lr);
+}
+
+void Adam::setProperty(const PropertyType type, const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::beta1:
+ status = setDouble(beta1, value);
+ break;
+ case PropertyType::beta2:
+ status = setDouble(beta2, value);
+ break;
+ case PropertyType::epsilon:
+ status = setDouble(epsilon, value);
+ break;
+ default:
+ Optimizer::setProperty(type, value);
+ status = ML_ERROR_NONE;
+ break;
+ }
+
+ throw_status(status);
+}
+
+void Adam::read(std::ifstream &file) {
+ OptType loaded_type;
+ file.read((char *)&loaded_type, sizeof(OptType));
+
+ if (loaded_type == type) {
+ if (continue_train) {
+ for (auto iter = weight_mv.begin(); iter != weight_mv.end(); iter++) {
+ (*iter).first.read(file);
+ (*iter).second.read(file);
+ }
+ } else {
+ size_t total_size = 0;
+ for (auto iter = weight_mv.begin(); iter != weight_mv.end(); iter++)
+ total_size += (*iter).first.getSize() + (*iter).second.getSize();
+
+ file.seekg(total_size, std::ifstream::cur);
+ }
+ } else {
+ ml_logw("Not loading saved optimizer parameters due to mismatched type");
+ }
+}
+
+void Adam::save(std::ofstream &file) {
+ Optimizer::save(file);
+
+ for (auto iter = weight_mv.begin(); iter != weight_mv.end(); iter++) {
+ (*iter).first.save(file);
+ (*iter).second.save(file);
+ }
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file adam.h
+ * @date 6 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the Adam optimizer.
+ */
+#ifndef __ADAM_H__
+#define __ADAM_H__
+#ifdef __cplusplus
+
+#include <optimizer_internal.h>
+
+namespace nntrainer {
+
+/**
+ * @class Adam optimizer class
+ * @brief Adam optimizer
+ */
+class Adam : public Optimizer {
+public:
+ /**
+ * @brief Constructor of Optimizer Class
+ */
+ template <typename... Args>
+ Adam(float lr = 0.001f, double b1 = 0.9f, double b2 = 0.999f,
+ double ep = 1.0e-7f, Args... args) :
+ Optimizer(OptType::ADAM, lr, args...),
+ beta1(b1),
+ beta2(b2),
+ epsilon(ep) {}
+
+ /**
+ * @copydoc apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ * int iteration)
+ */
+ void apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ int iteration);
+
+ /**
+ * @brief get the base name for the optimizer
+ * @retval base name of the optimizer
+ */
+ std::string getBaseName() { return "Adam"; };
+
+ /**
+ * @copydoc getLearningRate(int iteration)
+ */
+ double getLearningRate(int iteration);
+
+ /**
+ * @copydoc setProperty(const PropertyType type,
+ const std::string &value = "")
+ */
+ void setProperty(const PropertyType type, const std::string &value = "");
+
+ /**
+ * @copydoc Optimizer::initialize(std::shared_ptr<Weight> params, unsigned int
+ num_weights, bool setTensor)
+ */
+ int initialize(std::shared_ptr<Weight> params, unsigned int num_weights,
+ bool setTensor);
+
+ /**
+ * @copydoc read(std::ifstream &file)
+ */
+ void read(std::ifstream &file);
+
+ /**
+ * @copydoc save(std::ofstream &file)
+ */
+ void save(std::ofstream &file);
+
+ /**
+ * @brief get beta1
+ */
+ double getBeta1() { return beta1; };
+
+ /**
+ * @brief get beta2
+ */
+ double getBeta2() { return beta2; };
+
+ /**
+ * @brief get epsilon
+ */
+ double getEpsilon() { return epsilon; }
+
+private:
+ /**
+ * @brief Internal Tensors for adam Optimizer
+ */
+ std::vector<std::pair<Tensor, Tensor>> weight_mv;
+
+ double beta1; /** momentum for grad */
+ double beta2; /** momentum for grad**2 */
+ double epsilon; /** epsilon to protect overflow */
+};
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __ADAM_H__ */
--- /dev/null
+optimizer_sources = [
+ 'adam.cpp',
+ 'optimizer.cpp',
+ 'optimizer_factory.cpp',
+ 'sgd.cpp'
+]
+
+optimizer_headers = [
+ 'optimizer_factory.h',
+ 'optimizer_internal.h'
+]
+
+foreach s : optimizer_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
+
+foreach h : optimizer_headers
+ nntrainer_headers += join_paths(meson.current_source_dir(), h)
+endforeach
+
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file optimizer.cpp
+ * @date 08 April 2020
+ * @brief This is Implementation of Optimizer class
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <cmath>
+#include <fstream>
+#include <iostream>
+
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <optimizer_internal.h>
+#include <parse_util.h>
+#include <util_func.h>
+
+namespace nntrainer {
+
+int Optimizer::initialize(std::shared_ptr<Weight> weight_list,
+ unsigned int num_weights, bool set_tensor) {
+ return ML_ERROR_NONE;
+}
+
+double Optimizer::getLearningRate(int iteration) {
+ double ll = learning_rate;
+
+ if (decay_steps != 0) {
+ ll = ll * pow(decay_rate, (iteration / (float)decay_steps));
+ }
+
+ return ll;
+}
+
+void Optimizer::apply_gradients(std::shared_ptr<Weight> weight_list,
+ unsigned int num_weights, int iteration) {
+
+ double ll = getLearningRate(iteration);
+
+ int idx = 0;
+ for (unsigned int i = 0; i < num_weights; ++i) {
+ Weight &weight = weight_list.get()[i];
+
+ if (!weight.getTrainable())
+ continue;
+
+ apply_gradient(weight, idx, ll, iteration);
+ idx += 1;
+ }
+}
+
+int Optimizer::setProperty(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+
+ status = getKeyValue(values[i], key, value);
+ NN_RETURN_STATUS();
+
+ unsigned int type = parseOptProperty(key);
+
+ if (value.empty()) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ try {
+ /// @note this calls derived setProperty if available
+ setProperty(static_cast<PropertyType>(type), value);
+ } catch (...) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ }
+
+ try {
+ checkValidation();
+ } catch (...) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ return status;
+}
+
+void Optimizer::checkValidation() {
+ if (learning_rate <= 0.0f)
+ throw std::invalid_argument("Learning rate must be positive");
+}
+
+void Optimizer::setProperty(const PropertyType type, const std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::learning_rate:
+ status = setFloat(learning_rate, value);
+ break;
+ case PropertyType::decay_steps:
+ status = setUint(decay_steps, value);
+ break;
+ case PropertyType::decay_rate:
+ status = setFloat(decay_rate, value);
+ break;
+ case PropertyType::continue_train:
+ status = setBoolean(continue_train, value);
+ break;
+ default:
+ ml_loge("Error: Unknown Optimizer Property Key");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+
+ throw_status(status);
+}
+
+void Optimizer::read(std::ifstream &file) {
+ OptType loaded_type;
+ file.read((char *)&loaded_type, sizeof(OptType));
+
+ if (loaded_type >= OptType::UNKNOWN)
+ throw std::runtime_error("Saved file has unknown optimizer");
+}
+
+void Optimizer::save(std::ofstream &file) {
+ if (type >= OptType::UNKNOWN)
+ throw std::runtime_error("Cannot save unknown optimizer");
+
+ file.write((char *)&type, sizeof(OptType));
+}
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file optimizer_factory.cpp
+ * @date 7 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the optimizer factory.
+ */
+
+#include <adam.h>
+#include <optimizer_factory.h>
+#include <sgd.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Factory creator with copy constructor
+ */
+std::unique_ptr<Optimizer> createOptimizer(OptType type, const Optimizer &opt) {
+ switch (type) {
+ case OptType::SGD:
+ return std::make_unique<SGD>(static_cast<const SGD &>(opt));
+ case OptType::ADAM:
+ return std::make_unique<Adam>(static_cast<const Adam &>(opt));
+ case OptType::UNKNOWN:
+ /** fallthrough intended */
+ default:
+ throw std::invalid_argument("Unknown type for the optimizer");
+ }
+}
+
+/**
+ * @brief Factory creator with constructor
+ */
+std::unique_ptr<Optimizer> createOptimizer(OptType type) {
+ switch (type) {
+ case OptType::SGD:
+ return std::make_unique<SGD>();
+ case OptType::ADAM:
+ return std::make_unique<Adam>();
+ case OptType::UNKNOWN:
+ /** fallthrough intended */
+ default:
+ throw std::invalid_argument("Unknown type for the optimizer");
+ }
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file optimizer_factory.h
+ * @date 7 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the optimizer factory.
+ */
+
+#ifndef __OPTIMIZER_FACTORY_H__
+#define __OPTIMIZER_FACTORY_H__
+#ifdef __cplusplus
+
+#include <optimizer_internal.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Factory creator with copy constructor
+ */
+std::unique_ptr<Optimizer> createOptimizer(OptType type, const Optimizer &opt);
+
+/**
+ * @brief Factory creator with constructor
+ */
+std::unique_ptr<Optimizer> createOptimizer(OptType type);
+
+} // namespace nntrainer
+
+#endif // __cplusplus
+#endif // __OPTIMIZER_FACTORY_H__
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file optimizer_internal.h
+ * @date 08 April 2020
+ * @brief This is Optimizer classes of Neural Network
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+#ifndef __OPTIMIZER_H__
+#define __OPTIMIZER_H__
+#ifdef __cplusplus
+
+#include <memory>
+#include <optimizer.h>
+#include <tensor.h>
+#include <weight.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Enumeration of Optimizer
+ * 0. SGD
+ * 1. ADAM
+ * 2. Unknown
+ */
+using OptType = ml::train::OptimizerType;
+
+/**
+ * @class Optimizer Base class for optimizers
+ * @brief Base class for all optimizers
+ */
+class Optimizer : public ml::train::Optimizer {
+
+ /** Allow layer to initialize optimizer with itself */
+ friend class Layer;
+
+public:
+ /**
+ * @brief Default Constructor of Optimizer Class
+ */
+ Optimizer(const OptType t, float lr, float decay_rate = 1.0f,
+ unsigned int decay_steps = 0, float continue_train = false) :
+ type(t),
+ learning_rate(lr),
+ decay_rate(decay_rate),
+ decay_steps(decay_steps),
+ continue_train(continue_train) {
+ checkValidation();
+ }
+
+ /**
+ * @brief copy constructor
+ * @param[in] rhs Optimizer to be copied
+ */
+ Optimizer(const Optimizer &rhs) = default;
+
+ /**
+ * @brief copy assignment operator
+ * @param[in] rhs Optimizer to be copied
+ */
+ Optimizer &operator=(const Optimizer &rhs) = default;
+
+ /**
+ * @brief Move constructor of Conv 2D Layer.
+ * @param[in] Conv2dLayer &&
+ */
+ Optimizer(Optimizer &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs Optimizer to be moved.
+ */
+ Optimizer &operator=(Optimizer &&rhs) = default;
+
+ /**
+ * @brief get Optimizer Type
+ * @retval Optimizer type
+ */
+ OptType getType() { return type; };
+
+ /**
+ * @brief get Learning Rate
+ * @retval Learning rate
+ */
+ float getLearningRate() { return learning_rate; };
+
+ /**
+ * @brief get Decay Rate for learning rate decay
+ * @retval decay rate
+ */
+ float getDecayRate() { return decay_rate; };
+
+ /**
+ * @brief get Decay Steps for learning rate decay
+ * @retval decay steps
+ */
+ float getDecaySteps() { return decay_steps; };
+
+ /**
+ * @brief set Optimizer Parameters
+ * @param[in] values Optimizer Parameter list
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(std::vector<std::string> values);
+
+ /**
+ * @brief apply gradient to weight_list
+ * @param[in] params Weight list
+ * @param[in] num_weights size of the array
+ * @param[in] iteration nth epoch number
+ */
+ void apply_gradients(std::shared_ptr<Weight> params, unsigned int num_weights,
+ int iteration);
+
+ /**
+ * @brief Read Training optimizer paramters from file
+ * @param[in] file input stream file
+ */
+ virtual void read(std::ifstream &file);
+
+ /**
+ * @brief Save Training optimizer paramters from file
+ * @param[in] file output stream file
+ */
+ virtual void save(std::ofstream &file);
+
+ /**
+ * @brief setProperty by PropertyType
+ * @note By passing empty string, this can validate if @a type is valid
+ * @param[in] type property type to be passed
+ * @param[in] value value to be passed, if empty string is passed, do nothing
+ * but throws error when @a type is invalid
+ * @exception exception::not_supported when property type is not valid for
+ * the particular layer
+ * @exception std::invalid_argument invalid argument
+ */
+ virtual void setProperty(const PropertyType type,
+ const std::string &value = "");
+
+ /**
+ * @brief get the base name for the optimizer
+ * @retval base name of the optimizer
+ */
+ virtual std::string getBaseName() = 0;
+
+ /**
+ * @brief validate the optimizer
+ */
+ virtual void checkValidation();
+
+protected:
+ /**
+ * @brief Optimizer Type
+ */
+ OptType type;
+
+ /**
+ * @brief get Learning Rate for the given iteration
+ * @param[in] iteration Iteration for the learning rate
+ * @retval Learning rate
+ */
+ virtual double getLearningRate(int iteration);
+
+ float learning_rate; /** learning rate */
+ float decay_rate; /** decay rate for learning rate */
+ unsigned int decay_steps; /** decay steps for learning rate */
+ bool continue_train; /** Continue training with previous tensors for adam */
+
+private:
+ /**
+ * @brief initialize optimizer. Initialize Weight if it is adam
+ * @param[in] params Weight list
+ * @param[in] num_weights size of the array
+ * @param[in] setTensor true if the layer need weight update.
+ * Input Layer and Batch Normalization layer won't need it.
+ * Therefore, it sets false.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int initialize(std::shared_ptr<Weight> params,
+ unsigned int num_weights, bool setTensor);
+
+ /**
+ * @brief apply gradient to the given weight
+ * @param[in] weight Weight and gradient set to be updated
+ * @param[in] tensor_idx Idx of this tensor in the tensors list
+ * @param[in] num_weights size of the array
+ * @param[in] iteration nth epoch number
+ * @note weight which is called upon can be assumed to be trainable
+ */
+ virtual void apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ int iteration) = 0;
+};
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __OPTIMIZER_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file sgd.cpp
+ * @date 6 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the SGD optimizer.
+ */
+
+#include <sgd.h>
+
+namespace nntrainer {
+
+void SGD::apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ int iteration) {
+ Tensor &x = weight.getVariableRef();
+ const Tensor &x_grad = weight.getGradientRef();
+ x.add_i(x_grad, -updated_lr);
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file sgd.h
+ * @date 6 October 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is the SGD optimizer.
+ */
+#ifndef __SGD_H__
+#define __SGD_H__
+#ifdef __cplusplus
+
+#include <optimizer_internal.h>
+
+namespace nntrainer {
+
+/**
+ * @class SGD optimizer class
+ * @brief Stochastic Gradient Descent optimizer class
+ */
+class SGD : public Optimizer {
+public:
+ /**
+ * @brief Constructor of Optimizer Class
+ */
+ template <typename... Args>
+ SGD(float lr = 0.0001f, Args... args) :
+ Optimizer(OptType::SGD, lr, args...) {}
+
+ /**
+ * @copydoc apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ * int iteration)
+ */
+ void apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
+ int iteration);
+
+ /**
+ * @brief get the base name for the optimizer
+ * @retval base name of the optimizer
+ */
+ std::string getBaseName() { return "SGD"; };
+};
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __SGD_H__ */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jihoon Lee <jhoon.it.lee@samsung.com>
- *
- * @file activation_layer.cpp
- * @date 17 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jihoon Lee <jhoon.it.lee@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Activation Layer Class for Neural Network
- *
- */
-
-#include <algorithm>
-#include <cmath>
-#include <fstream>
-#include <functional>
-#include <iostream>
-#include <vector>
-
-#include <activation_layer.h>
-#include <blas_interface.h>
-#include <layer_internal.h>
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <optimizer_internal.h>
-#include <parse_util.h>
-#include <tensor.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-/**
- * @brief Initialize the layer
- *
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
-int ActivationLayer::initialize() {
-
- output_dim = input_dim;
-
- return ML_ERROR_NONE;
-}
-
-sharedConstTensors ActivationLayer::forwarding(sharedConstTensors in) {
- input = *in[0];
- /// @note @a _act_fn is expected to work out of place and not modify @a input
- hidden = _act_fn(input);
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors ActivationLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- Tensor deriv = *derivative[0];
- Tensor ret;
- if (activation_type == ActivationType::ACT_SOFTMAX)
- ret = _act_prime_fn(hidden, deriv);
- else
- ret = _act_prime_fn(input, deriv);
-
- return {MAKE_SHARED_TENSOR(std::move(ret))};
-}
-
-int ActivationLayer::setActivation(
- std::function<Tensor(Tensor const &)> const &activation_fn,
- std::function<Tensor(Tensor const &, Tensor const &)> const
- &activation_prime_fn) {
- _act_fn = activation_fn;
- _act_prime_fn = activation_prime_fn;
-
- return ML_ERROR_NONE;
-}
-
-int ActivationLayer::setActivation(
- std::function<Tensor(Tensor const &)> const &activation_fn,
- std::function<Tensor(Tensor const &)> const &activation_prime_fn) {
- _act_fn = activation_fn;
- _act_prime_fn = [activation_prime_fn](Tensor const &x,
- Tensor const &derivative) {
- return derivative.multiply(activation_prime_fn(x));
- };
-
- return ML_ERROR_NONE;
-}
-
-int ActivationLayer::setActivation(
- std::function<float(float const)> const &activation_fn,
- std::function<float(float const)> const &activation_prime_fn) {
- _act_fn = [activation_fn](Tensor const &x) { return x.apply(activation_fn); };
- _act_prime_fn = [activation_prime_fn](Tensor const &x,
- Tensor const &derivative) {
- return derivative.multiply(x.apply(activation_prime_fn));
- };
-
- return ML_ERROR_NONE;
-}
-
-/**
- * @brief setActivation by preset ActivationType
- *
- * @param[in] ActivationType ActivationType ActivationType to be set
- */
-void ActivationLayer::setActivation(ActivationType acti_type) {
- Layer::setActivation(acti_type);
-
- switch (acti_type) {
- case ActivationType::ACT_TANH:
- this->setActivation(tanhFloat, tanhPrime);
- break;
- case ActivationType::ACT_SIGMOID:
- this->setActivation(sigmoid, sigmoidPrime);
- break;
- case ActivationType::ACT_SOFTMAX:
- this->setActivation(softmax, softmaxPrime);
- break;
- case ActivationType::ACT_RELU:
- this->setActivation(relu, reluPrime);
- break;
- case ActivationType::ACT_NONE:
- this->setActivation(no_op, no_op_prime);
- break;
- case ActivationType::ACT_UNKNOWN:
- default:
- throw std::runtime_error("Error: Not Supported Activation Type");
- }
-}
-
-Tensor ActivationLayer::softmax(Tensor const &t) {
- /**
- * shiftx_logit = logit - max_batch(logit)
- * softmax = exp(shiftx_logit) / (sum(exp(shiftx_logit)))
- */
- unsigned int batch = t.batch();
- float *dp;
- float *rp;
-
- Tensor divisor = t.clone();
-
- dp = divisor.getData();
- unsigned int feat_len = t.getDim().getFeatureLen();
-
- for (unsigned int k = 0; k < batch; k++) {
- int index = k * feat_len;
- // find max and subtract it
- float m = *std::max_element(dp + index, dp + index + feat_len);
-
- Tensor tmp = Tensor(1, 1, 1, feat_len);
- tmp.setValue(m);
- saxpy(feat_len, -1, tmp.getData(), 1, dp + index, 1);
- }
-
- // take exp
- Tensor result = divisor.apply(exp_util);
- rp = result.getData();
- // take sum over batch
- Tensor sum = result.sum_by_batch();
-
- for (unsigned int k = 0; k < batch; k++) {
- int index = k * feat_len;
- std::transform(rp + index, rp + index + feat_len, rp + index,
- std::bind(std::divides<float>(), std::placeholders::_1,
- sum.getValue(k, 0, 0, 0)));
- }
-
- return result;
-}
-
-Tensor ActivationLayer::softmaxPrime(Tensor const &x,
- Tensor const &derivative) {
- unsigned int batch = x.batch();
- unsigned int channel = x.channel();
- unsigned int height = x.height();
- unsigned int width = x.width();
- bool is_derivative = true;
-
- Tensor PI = Tensor(x.getDim());
-
- const float *xp = x.getData();
- const float *d = derivative.getData();
- float *pp = PI.getData();
-
- /** @todo update default tensorDim to be 0 and not 1 */
- if (derivative.getDim() == TensorDim()) {
- is_derivative = false;
- }
-
- for (unsigned int k = 0; k < batch; ++k) {
- int K = k * channel * height * width;
- for (unsigned int c = 0; c < channel; ++c) {
- int C = K + c * height * width;
- for (unsigned int i = 0; i < height; ++i) {
- int I = C + i * width;
- for (unsigned int j = 0; j < width; ++j) {
- float sum = 0.0f;
- for (unsigned int l = 0; l < width; ++l) {
- float val;
- if (j == l) {
- val = xp[I + l] * (1.0f - xp[I + j]);
- } else {
- val = -xp[I + l] * xp[I + j];
- }
- if (is_derivative)
- val *= d[I + l];
- sum += val;
- }
- pp[I + j] = sum;
- }
- }
- }
- }
- return PI;
-}
-
-float ActivationLayer::sigmoid(float x) { return 1.0f / (1.0f + exp_util(-x)); }
-
-float ActivationLayer::sigmoidPrime(float x) {
- float sprime = sigmoid(x);
- return sprime * (1.0f - sprime);
-}
-
-float ActivationLayer::tanhFloat(float x) { return (float)tanh(x); }
-
-float ActivationLayer::tanhPrime(float x) {
- float th = (float)tanh(x);
- return 1.0f - th * th;
-}
-
-float ActivationLayer::relu(float x) {
- if (x <= 0.0f) {
- return 0.0f;
- } else {
- return x;
- }
-}
-
-float ActivationLayer::reluPrime(float x) {
- if (x <= 0.0f) {
- return 0.0f;
- } else {
- return 1.0f;
- }
-}
-
-float ActivationLayer::no_op(float x) { return x; }
-
-float ActivationLayer::no_op_prime(float x) { return 1.0f; }
-}; // namespace nntrainer
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file adam.cpp
- * @date 6 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the Adam optimizer.
- */
-
-#include <cmath>
-#include <fstream>
-
-#include <adam.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int Adam::initialize(std::shared_ptr<Weight> weight_list,
- unsigned int num_weights, bool set_tensor) {
- int status = ML_ERROR_NONE;
- weight_mv.clear();
-
- if (set_tensor) {
- for (unsigned int i = 0; i < num_weights; ++i) {
- Weight &w = weight_list.get()[i];
-
- // TODO: only trainable weights must be sent to optimizer
- if (!w.getTrainable())
- continue;
-
- Tensor m = Tensor(w.getDim());
- m.setZero();
- Tensor v = Tensor(w.getDim());
- v.setZero();
- std::pair<Tensor, Tensor> p =
- std::pair<Tensor, Tensor>(std::move(m), std::move(v));
- weight_mv.push_back(std::move(p));
- }
- }
- return status;
-}
-
-double Adam::getLearningRate(int iteration) {
- double ll = Optimizer::getLearningRate(iteration);
-
- std::function<float(double)> biasCorrection = [&](float f) {
- return 1.0f - pow(f, iteration + 1);
- };
-
- ll *= sqrt(biasCorrection(beta2)) / biasCorrection(beta1);
-
- return ll;
-}
-
-void Adam::apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- int iteration) {
-
- Tensor &x = weight.getVariableRef();
- const Tensor &x_grad = weight.getGradientRef();
-
- // This is implementation of adam from original paper.
- // This is not deleted intentionally.
- // float biasCorrection1 = 1 - pow(beta1, iteration + 1);
- // float biasCorrection2 = 1 - pow(beta2, iteration + 1);
- // Tensor &wm = weight_mv[idx].first;
- // Tensor &wv = weight_mv[idx].second;
-
- // wm.multiply_i(beta1);
- // wm.add_i(x_grad, 1.0f - beta1);
-
- // wv.multiply_i(beta2);
- // wv.add_i(x_grad.multiply(x_grad), 1.0f - beta2);
-
- // Tensor denom = wv.apply(sqrtFloat)
- // .divide(sqrtFloat(biasCorrection2))
- // .add(epsilon);
- // x.add_i(wm.divide(denom), -ll / biasCorrection1);
-
- std::function<double(double)> sqrtEps = [&](double f) {
- return sqrtDouble(f) + this->epsilon;
- };
-
- Tensor &wm = weight_mv[tensor_idx].first;
- Tensor &wv = weight_mv[tensor_idx].second;
-
- wm.multiply_i(beta1);
- wm.add_i(x_grad, 1.0f - beta1);
-
- wv.multiply_i(beta2);
- wv.add_i(x_grad.multiply(x_grad), 1.0f - beta2);
-
- x.add_i(wm.divide(wv.apply(sqrtEps)), -updated_lr);
-}
-
-void Adam::setProperty(const PropertyType type, const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::beta1:
- status = setDouble(beta1, value);
- break;
- case PropertyType::beta2:
- status = setDouble(beta2, value);
- break;
- case PropertyType::epsilon:
- status = setDouble(epsilon, value);
- break;
- default:
- Optimizer::setProperty(type, value);
- status = ML_ERROR_NONE;
- break;
- }
-
- throw_status(status);
-}
-
-void Adam::read(std::ifstream &file) {
- OptType loaded_type;
- file.read((char *)&loaded_type, sizeof(OptType));
-
- if (loaded_type == type) {
- if (continue_train) {
- for (auto iter = weight_mv.begin(); iter != weight_mv.end(); iter++) {
- (*iter).first.read(file);
- (*iter).second.read(file);
- }
- } else {
- size_t total_size = 0;
- for (auto iter = weight_mv.begin(); iter != weight_mv.end(); iter++)
- total_size += (*iter).first.getSize() + (*iter).second.getSize();
-
- file.seekg(total_size, std::ifstream::cur);
- }
- } else {
- ml_logw("Not loading saved optimizer parameters due to mismatched type");
- }
-}
-
-void Adam::save(std::ofstream &file) {
- Optimizer::save(file);
-
- for (auto iter = weight_mv.begin(); iter != weight_mv.end(); iter++) {
- (*iter).first.save(file);
- (*iter).second.save(file);
- }
-}
-
-} // namespace nntrainer
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file addition_layer.cpp
- * @date 30 July 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Addition Layer Class for Neural Network
- *
- */
-
-#include <addition_layer.h>
-#include <layer_internal.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int AdditionLayer::initialize() {
- int status = ML_ERROR_NONE;
- if (num_inputs == 0) {
- ml_loge("Error: number of inputs are not initialized");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- for (unsigned int idx = 0; idx < num_inputs; ++idx) {
- if (input_dim[idx].getDataLen() == 1) {
- ml_logw("Warning: the length of previous layer dimension is one");
- }
- }
-
- /** input dimension indicates the dimension for all the inputs to follow */
- output_dim[0] = input_dim[0];
-
- return status;
-}
-
-sharedConstTensors AdditionLayer::forwarding(sharedConstTensors in) {
- hidden = Tensor(input_dim[0]);
- hidden.setZero();
-
- TensorDim &in_dim = input_dim[0];
-
- for (unsigned int idx = 0; idx < num_inputs; ++idx) {
- if (in_dim != in[idx]->getDim())
- throw std::runtime_error("Error: addition layer requires same "
- "shape from all input layers");
- hidden.add_i(*in[idx]);
- }
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors AdditionLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
-
- sharedConstTensors ret;
- for (unsigned int i = 0; i < num_inputs; ++i) {
- sharedTensor t =
- std::shared_ptr<Tensor>(new Tensor(), std::default_delete<Tensor[]>());
- *t = *derivative[0];
- ret.push_back(t);
- }
-
- return ret;
-}
-
-void AdditionLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::num_inputs: {
- if (!value.empty()) {
- status = setUint(num_inputs, value);
- throw_status(status);
- if (num_inputs < 1)
- throw std::invalid_argument("Minimum number of inputs must be 1");
- }
- } break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file blas_interface.cpp
- * @date 28 Aug 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is dummy header for blas support
- *
- */
-
-#include <blas_interface.h>
-#include <nntrainer_error.h>
-
-#include <cmath>
-
-#define sgemv_loop(ci, cj, cM, cN) \
- do { \
- double y0; \
- unsigned int i, j; \
- for (ci = 0; ci != cM; ci++) { \
- y0 = Y[ci * incy] * beta; \
- for (cj = 0; cj != cN; cj++) \
- y0 += A[i + j * lda] * X[cj * incx]; \
- Y[ci * incy] = y0; \
- } \
- } while (0);
-
-namespace nntrainer {
-
-#ifndef USE_BLAS
-
-static void saxpy_raw(const unsigned int N, const float alpha, const float *X,
- const int incX, float *Y, const int incY) {
- if (incX <= 0 or incY <= 0)
- throw std::invalid_argument(
- "Error: negative inc not supported without cblas");
- for (unsigned int i = 0; i < N; ++i)
- Y[i * incY] = Y[i * incY] + X[i * incX] * alpha;
-}
-
-static void sgemv_raw(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA,
- const unsigned int M, const unsigned int N,
- const float alpha, const float *A, const unsigned int lda,
- const float *X, const int incX, const float beta,
- float *Y, const int incY) {
-
- unsigned int incy = abs(incY);
- unsigned int incx = abs(incX);
- if (TransA == CblasTrans) {
- sgemv_loop(i, j, N, M);
- } else {
- sgemv_loop(j, i, M, N);
- }
-}
-
-static void scopy_raw(const unsigned int N, const float *X, const int incX,
- float *Y, const int incY) {
- unsigned int incy = abs(incY);
- unsigned int incx = abs(incX);
-
- for (unsigned int i = 0; i < N; ++i)
- Y[i * incy] = X[i * incx];
-}
-
-static void sscal_raw(const unsigned int N, const float alpha, float *X,
- const int incX) {
- unsigned int incx = abs(incX);
-
- for (unsigned int i = 0; i < N; ++i)
- X[i * incx] = alpha * X[i * incx];
-}
-
-static float snrm2_raw(const unsigned int N, const float *X, const int incX) {
- unsigned int incx = abs(incX);
- float sum = 0.0f;
- float tmp;
-#pragma omp parallel for private(tmp) reduction(+ : sum)
- for (unsigned int i = 0; i < N; i++) {
- tmp = X[i * incx];
- sum += tmp * tmp;
- }
- return sqrt(sum);
-}
-
-static void sgemm_raw(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA,
- CBLAS_TRANSPOSE TransB, const unsigned int M,
- const unsigned int N, const unsigned int K,
- const float alpha, const float *A, const unsigned int lda,
- const float *B, const unsigned int ldb, const float beta,
- float *C, const unsigned int ldc) {
-
- for (unsigned int m = 0; m < M; ++m) {
- for (unsigned int n = 0; n < N; ++n) {
- double c = 0.0;
- float c_old = C[m * ldc + n];
- for (unsigned int k = 0; k < K; ++k) {
- float a, b;
- a = ((TransA == CblasTrans) ? A[k * lda + m] : A[m * lda + k]);
- b = ((TransB == CblasTrans) ? B[n * ldb + k] : B[k * ldb + n]);
- c += a * b;
- }
- C[m * ldc + n] = alpha * c;
- if (beta != 0.0)
- C[m * ldc + n] += beta * c_old;
- }
- }
-}
-
-#endif
-
-void saxpy(const unsigned int N, const float alpha, const float *X,
- const int incX, float *Y, const int incY) {
-#ifdef USE_BLAS
- cblas_saxpy(N, alpha, X, incX, Y, incY);
-#else
- saxpy_raw(N, alpha, X, incX, Y, incY);
-#endif
-}
-
-void sgemm(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, CBLAS_TRANSPOSE TransB,
- const unsigned int M, const unsigned int N, const unsigned int K,
- const float alpha, const float *A, const unsigned int lda,
- const float *B, const unsigned int ldb, const float beta, float *C,
- const unsigned int ldc) {
-
-#ifdef USE_CUBLAS
- int devID = 0;
- cudaDeviceProp deviceProp;
- cudaGetDeviceProperties(&deviceProp, devID);
- float *d_A, *d_B, *d_C;
-
- unsigned int size_A = M * K * sizeof(float);
- unsigned int size_B = K * N * sizeof(float);
- unsigned int size_C = M * N * sizeof(float);
-
- cudaMalloc((void **)&d_A, size_A);
- cudaMalloc((void **)&d_B, size_B);
- cudaMemcpy(d_A, A, size_A, cudaMemcpyHostToDevice);
- cudaMemcpy(d_B, B, size_B, cudaMemcpyHostToDevice);
- cudaMalloc((void **)&d_C, size_C);
-
- cublasHandle_t handle;
- cublasCreate(&handle);
-
- cublasOperation_t transA = (TransA == CblasTrans) ? CUBLAS_OP_T : CUBLAS_OP_N;
- cublasOperation_t transB = (TransB == CblasTrans) ? CUBLAS_OP_T : CUBLAS_OP_N;
- cublasSgemm(handle, transA, transB, N, M, K, &alpha, d_B, N, d_A, K, &beta,
- d_C, N);
-
- cudaMemcpy(C, d_C, size_C, cudaMemcpyDeviceToHost);
- cublasDestroy(handle);
-#elif defined USE_BLAS
- cblas_sgemm(order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C,
- ldc);
-#else
- sgemm_raw(order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C,
- ldc);
-#endif
-}
-
-void scopy(const unsigned int N, const float *X, const int incX, float *Y,
- const int incY) {
-#ifdef USE_BLAS
- cblas_scopy(N, X, incX, Y, incY);
-#else
- scopy_raw(N, X, incX, Y, incY);
-#endif
-}
-
-void sscal(const int N, const float alpha, float *X, const int incX) {
-#ifdef USE_BLAS
- cblas_sscal(N, alpha, X, incX);
-#else
- sscal_raw(N, alpha, X, incX);
-#endif
-}
-
-float snrm2(const int N, const float *X, const int incX) {
-#ifdef USE_BLAS
- return cblas_snrm2(N, X, incX);
-#else
- return snrm2_raw(N, X, incX);
-#endif
-}
-
-void sgemv(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, const unsigned int M,
- const unsigned int N, const float alpha, const float *A,
- const unsigned int lda, const float *X, const int incX,
- const float beta, float *Y, const int incY) {
-#ifdef USE_BLAS
- return cblas_sgemv(order, TransA, M, N, alpha, A, lda, X, incX, beta, Y,
- incY);
-#else
- return sgemv_raw(order, TransA, M, N, alpha, A, lda, X, incX, beta, Y, incY);
-#endif
-}
-
-} // namespace nntrainer
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file bn_layer.cpp
- * @date 14 May 2020
- * @brief This is Batch Normalization Layer Class for Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <assert.h>
-#include <bn_layer.h>
-#include <layer_internal.h>
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-enum class BNParams { mu, var, gamma, beta };
-
-/// @todo add multiple axis support
-int BatchNormalizationLayer::initialize() {
- int status = ML_ERROR_NONE;
-
- if (num_inputs != 1) {
- throw std::invalid_argument(
- "Only one input is allowed for batch normalization layer");
- }
-
- output_dim[0] = input_dim[0];
- TensorDim dim;
-
- /// @note this logic cannot tell channel is actually 1 or it is just not used.
- if (axis == -1)
- axis = input_dim[0].channel() > 1 ? 1 : 3;
-
- dim.setTensorDim(axis, input_dim[0].getTensorDim(axis));
-
- for (int i = 0; i < 4; ++i) {
- if (axis != i)
- axes_to_reduce.push_back(i);
- }
-
- setNumWeights(4);
- weightAt(0) =
- std::move(Weight(dim, initializers[static_cast<int>(BNParams::mu)], false,
- "BN:moving_mean"));
- ///@todo shift var to std to save computation
- weightAt(1) =
- std::move(Weight(dim, initializers[static_cast<int>(BNParams::var)], false,
- "BN:moving_variance"));
- weightAt(2) = std::move(Weight(
- dim, initializers[static_cast<int>(BNParams::gamma)], true, "BN:gamma"));
- weightAt(3) = std::move(Weight(
- dim, initializers[static_cast<int>(BNParams::beta)], true, "BN:beta"));
-
- return status;
-}
-
-void BatchNormalizationLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
- switch (type) {
- case PropertyType::epsilon:
- if (!value.empty()) {
- status = setFloat(epsilon, value);
- throw_status(status);
- }
- break;
- case PropertyType::moving_mean_initializer:
- if (!value.empty()) {
- initializers[static_cast<int>(BNParams::mu)] =
- (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
- }
- break;
- case PropertyType::moving_variance_initializer:
- if (!value.empty()) {
- initializers[static_cast<int>(BNParams::var)] =
- (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
- }
- break;
- case PropertyType::beta_initializer:
- if (!value.empty()) {
- initializers[static_cast<int>(BNParams::beta)] =
- (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
- }
- break;
- case PropertyType::gamma_initializer:
- if (!value.empty()) {
- initializers[static_cast<int>(BNParams::gamma)] =
- (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
- }
- break;
- case PropertyType::momentum:
- if (!value.empty()) {
- status = setFloat(momentum, value);
- throw_status(status);
- }
- break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-sharedConstTensors BatchNormalizationLayer::forwarding(sharedConstTensors in) {
- Tensor &mu = weightAt(static_cast<int>(BNParams::mu)).getVariableRef();
- Tensor &var = weightAt(static_cast<int>(BNParams::var)).getVariableRef();
- Tensor &gamma = weightAt(static_cast<int>(BNParams::gamma)).getVariableRef();
- Tensor &beta = weightAt(static_cast<int>(BNParams::beta)).getVariableRef();
-
- input = *in[0];
- /// @todo change trainable #524
- if (trainable) {
- Tensor cmu = input.average(axes_to_reduce);
- deviation = input.subtract(cmu);
-
- cvar = deviation.pow(2.0f).average(axes_to_reduce);
-
- mu.multiply_i(momentum);
- mu.add_i(cmu, 1 - momentum);
- var.multiply_i(momentum);
- var.add_i(cvar, 1 - momentum);
-
- cvar.add_i(epsilon);
-
- invstd = cvar.pow(-0.5f);
- this->x_normalized = deviation.multiply(invstd);
- this->hidden = x_normalized.multiply(gamma);
- this->hidden.add_i(beta);
- } else {
- deviation = input.subtract(mu);
- this->x_normalized = deviation.divide(var.add(epsilon).pow(0.5f));
- this->hidden = x_normalized.multiply(gamma);
- this->hidden.add(beta);
- }
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors
-BatchNormalizationLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- Tensor &gamma = weightAt(static_cast<int>(BNParams::gamma)).getVariableRef();
- Tensor &dgamma = weightAt(static_cast<int>(BNParams::gamma)).getGradientRef();
- Tensor &dbeta = weightAt(static_cast<int>(BNParams::beta)).getGradientRef();
- Tensor dx_normalized;
-
- Tensor deriv = *derivative[0];
-
- int N = 1;
-
- for (auto &axis : axes_to_reduce) {
- N *= input_dim[0].getTensorDim(axis);
- }
-
- dbeta = deriv.sum(axes_to_reduce);
- dgamma = deviation.multiply(invstd).multiply(deriv).sum(axes_to_reduce);
-
- Tensor dx_1 = gamma.multiply(invstd);
- Tensor dx_2 = deriv.multiply(N);
- dx_2.subtract_i(deriv.sum(axes_to_reduce));
- dx_2.subtract_i(deviation.divide(cvar).multiply(
- deviation.multiply(deriv).sum(axes_to_reduce)));
-
- Tensor dx = dx_2.multiply(dx_1);
- dx.divide_i(N);
-
- opt->apply_gradients(weight_list, num_weights, iteration);
-
- return {MAKE_SHARED_TENSOR(std::move(dx))};
-}
-
-void BatchNormalizationLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<BatchNormalizationLayer> from =
- std::static_pointer_cast<BatchNormalizationLayer>(l);
- this->cvar.copy(from->cvar);
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file concat_layer.cpp
- * @date 27 Oct 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Concat Layer Class for Neural Network
- *
- */
-
-#include <concat_layer.h>
-#include <cstring>
-#include <layer_internal.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int ConcatLayer::initialize() {
- int status = ML_ERROR_NONE;
- unsigned int channel = 0;
-
- if (num_inputs == 0) {
- ml_loge("Error: number of inputs are not initialized");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- const TensorDim &d = input_dim[0];
- channel += d.channel();
- for (unsigned int idx = 1; idx < num_inputs; ++idx) {
- const TensorDim &dim = input_dim[idx];
-
- for (unsigned int i = 2; i < d.rank(); ++i) {
- if (d[i] != dim[i])
- throw std::runtime_error("Error: concat layer requires same "
- "shape from all input layers");
- }
- channel += input_dim[idx].channel();
- }
-
- output_dim[0] = input_dim[0];
- output_dim[0].channel(channel);
-
- return status;
-}
-
-sharedConstTensors ConcatLayer::forwarding(sharedConstTensors in) {
- hidden = Tensor(output_dim[0]);
-
-#ifdef DEBUG
- const TensorDim &d = in[0]->getDim();
- channel += d.channel();
- for (unsigned int idx = 1; idx < num_inputs; ++idx) {
- const TensorDim &dim = in[idx]->getDim();
-
- for (unsigned int i = 2; i < d.rank(); ++i) {
- if (d[i] != dim[i])
- throw std::runtime_error("Error: concat layer requires same "
- "shape from all input layers");
- }
- channel += input_dim[idx].channel();
- }
-
- if (channel != output_dim[0].channel())
- throw std::runtime_error(
- "Error: Sum of channel of input layers is not same with output channel");
-#endif
-
- unsigned int f_size = output_dim[0].getFeatureLen();
-
- for (unsigned int b = 0; b < input_dim[0].batch(); ++b) {
- unsigned int position = 0;
- for (unsigned int idx = 0; idx < num_inputs; ++idx) {
- TensorDim in_dim = in[idx]->getDim();
- memcpy(hidden.getAddress(b * f_size + position),
- in[idx]->getAddress(b * in_dim.getFeatureLen()),
- in_dim.getFeatureLen() * sizeof(float));
- position += in_dim.getFeatureLen();
- }
- }
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors ConcatLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- sharedConstTensors ret;
- TensorDim d = derivative[0]->getDim();
-
- unsigned int position = 0;
- for (unsigned int idx = 0; idx < num_inputs; ++idx) {
- TensorDim in_dim = input_dim[idx];
- sharedTensor t = std::shared_ptr<Tensor>(new Tensor(in_dim),
- std::default_delete<Tensor>());
-
- for (unsigned int b = 0; b < in_dim.batch(); ++b) {
- memcpy(t->getAddress(b * in_dim.getFeatureLen()),
- derivative[0]->getAddress(b * d.getFeatureLen() + position),
- in_dim.getFeatureLen() * sizeof(float));
- }
- position += in_dim.getFeatureLen();
-
- ret.push_back(t);
- }
-
- return ret;
-}
-
-void ConcatLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::num_inputs: {
- if (!value.empty()) {
- status = setUint(num_inputs, value);
- throw_status(status);
- if (num_inputs < 1)
- throw std::invalid_argument("Minimum number of inputs must be 1");
- }
- } break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file conv2d_layer.h
- * @date 02 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Convolution Layer Class for Neural Network
- *
- */
-#include <cstring>
-#include <string>
-
-#include <blas_interface.h>
-#include <conv2d_layer.h>
-#include <layer_internal.h>
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int Conv2DLayer::initialize() {
- int status = ML_ERROR_NONE;
-
- if (input_dim.size() != 1 || output_dim.size() != 1) {
- throw std::invalid_argument("Convolution layer only takes one input");
- }
-
- TensorDim &in_dim = input_dim[0];
- TensorDim &out_dim = output_dim[0];
-
- if (in_dim.getDataLen() == 1) {
- ml_logw("Warning: the length of previous layer dimension is one");
- }
-
- TensorDim dim =
- TensorDim(1, in_dim.channel(), kernel_size[0], kernel_size[1]);
- TensorDim bias_dim = TensorDim(1, 1, 1, 1);
-
- std::string kernelPrefix = "Conv2d:filter";
- std::string biasPrefix = "Conv2d:bias";
- setNumWeights(filter_size * 2);
-
- for (unsigned int i = 0; i < filter_size; ++i) {
- /*< @note: order of weight and bias are:
- w0 w1 w2 ... w3
- */
- weightAt(i) = std::move(
- Weight(dim, weight_initializer, true, kernelPrefix + std::to_string(i)));
- weightAt(i + filter_size) = std::move(
- Weight(bias_dim, bias_initializer, true, biasPrefix + std::to_string(i)));
- }
-
- // this output_dim should be the same with dimension of hidden
- out_dim.batch(in_dim.batch());
- out_dim.channel(filter_size);
- out_dim.height(
- (in_dim.height() - kernel_size[0] + 2 * padding[0]) / stride[0] + 1);
- out_dim.width((in_dim.width() - kernel_size[1] + 2 * padding[1]) / stride[1] +
- 1);
-
- return status;
-}
-
-void Conv2DLayer::read(std::ifstream &file) { Layer::read(file); }
-
-void Conv2DLayer::save(std::ofstream &file) { Layer::save(file); }
-
-sharedConstTensors Conv2DLayer::forwarding(sharedConstTensors in) {
- int status = ML_ERROR_NONE;
-
- if (num_inputs != 1)
- throw std::invalid_argument("Convolution layer only takes one input");
-
- input = *in[0];
-
- TensorDim &in_dim = input_dim[0];
- TensorDim &out_dim = output_dim[0];
-
- if (normalization) {
- input = input.normalization();
- }
-
- if (standardization) {
- input = input.standardization();
- }
-
- TensorDim hidden_dim = output_dim[0];
- hidden = Tensor(hidden_dim);
- hidden.setZero();
-
- /** Calculate Convolution 2D
- *
- * This is the 2D Matrix Shape [ height ] x [ width ]
- * . Height : filter_size
- * . Width : Input Channel * Kernel_size[0] * Kernel_size[1]
- *
- * imKernel
- * +------|------|------+
- * |------|------|------|
- * [filter_size (height)] |------|------|------|
- * |------|------|------|
- * +------|------|------+
- * [Input Channel * Kernel_size[0]
- * * Kernel_size[1] (width)]
- *
- *
- * After im2Col with channel_mode true (in : input)
- *
- * This is the 2D Matrix Shape [ height ] x [ width ]
- * . Height : Input Channel * Kernel_size[0] * Kernel_size[1]
- * . Width : output_dim.height * output_dim.width
- *
- * +-|-|-|-| |-|-|-|-+
- * [Input Channel | | | | | | | | | |
- * * Kernel_size[0] |_|_|_|_| |_|_|_|_|
- * * Kenel_size[1] | | | | | .... | | | | |
- * (height)] |_|_|_|_| |_|_|_|_|
- * | | | | | | | | | |
- * +_|_|_|_| |_|_|_|_+
- * [ output_dim.height
- * * output_dim.width (width) ]
- *
- * Output Dimention
- * -> [Channel ( = filter_size = output_dim.channel )]
- * x [output_dim.height x output_dim.width]
- */
-
- TensorDim kdim(filter_size, in_dim.channel(), kernel_size[0], kernel_size[1]);
-
- std::vector<float> imkernel(kdim.getFeatureLen() * filter_size);
-
- for (unsigned int i = 0; i < filter_size; ++i) {
- Tensor &filters = weightAt(i).getVariableRef();
- float *d = imkernel.data();
- memcpy(&d[i * kdim.getFeatureLen()], filters.getData(),
- kdim.getFeatureLen() * sizeof(float));
- }
-
- for (unsigned int b = 0; b < in_dim.batch(); ++b) {
- std::vector<float> out(out_dim.getFeatureLen());
- Tensor inSub(TensorDim(1, input.channel(), input.height(), input.width()),
- input.getAddress(b * input.getDim().getFeatureLen()));
-
- status = conv2d_gemm(imkernel.data(), kdim, inSub, out_dim, stride, padding,
- out.data(), out.size(), true);
- if (status != ML_ERROR_NONE)
- throw std::runtime_error("Forwarding Convolution failed.");
- memcpy(hidden.getAddress(b * hidden.getDim().getFeatureLen()), out.data(),
- out.size() * sizeof(float));
-
- for (unsigned int i = 0; i < filter_size; i++) {
- Tensor &bias = weightAt(i + filter_size).getVariableRef();
- Tensor tmp(1, 1, hidden.height(), hidden.width());
- tmp.setValue(bias.getValue(0, 0, 0, 0));
- saxpy(hidden.height() * hidden.width(), 1, tmp.getData(), 1,
- hidden.getAddress(b * hidden.getDim().getFeatureLen() +
- i * hidden.height() * hidden.width()),
- 1);
- }
- }
-
- loss = 0.0f;
- if (weight_regularizer == WeightRegularizerType::l2norm) {
- for (unsigned int i = 0; i < filter_size; ++i) {
- Tensor &weight = weightAt(i).getVariableRef();
- loss += weight_regularizer_constant * 0.5f * (weight.l2norm());
- }
- loss /= filter_size;
- }
-
- return {MAKE_SHARED_TENSOR(hidden)};
-};
-
-sharedConstTensors Conv2DLayer::backwarding(sharedConstTensors derivatives,
- int iteration) {
-
- TensorDim &in_dim = input_dim[0];
-
- std::array<unsigned int, CONV2D_DIM> same_pad;
- sharedConstTensor derivative = derivatives[0];
-
- same_pad[0] = kernel_size[0] - 1;
- same_pad[1] = kernel_size[1] - 1;
-
- for (unsigned int i = 0; i < filter_size; ++i) {
- Tensor &delK = weightAt(i).getGradientRef();
- Tensor &delBias = weightAt(i + filter_size).getGradientRef();
- delK.setZero();
- delBias.setZero();
- }
-
- Tensor ret(in_dim.batch(), in_dim.channel(), in_dim.height() + padding[0] * 2,
- in_dim.width() + padding[1] * 2);
- ret.setZero();
-
- /** Calculate DelK
- *
- * This is the 2D Matrix Shape [ height ] x [ width ]
- * . Height : filter_size
- * . Width : derivative.height * derivative.width
- *
- * derivative
- * +------|------+
- * |------|------|
- * [filter_size (height) |------|------|
- * (=derivative->channel) |------|------|
- * +------|------+
- * [derivative->height
- * * derivative->width (width)]
- *
- *
- * After im2Col with channel_mode false ( in : input )
- *
- * This is the 2D Matrix Shape [ height ] x [ width ]
- * . Height : derivative.height * derivative.width
- * . Width : input_dim.channel * Kernel_size[0] * Kernel_size[1]
- *
- * +-|-|-|-| |-|-|-|-+
- * | | | | | | | | | |
- * [derivative->width |_|_|_|_| |_|_|_|_|
- * * derivative->height | | | | | .... | | | | |
- * (height)] +_|_|_|_| |_|_|_|_+
- * [ input_dim.channel * kernel_size[0]
- * * kernel_size[1] (width) ]
- *
- * Output Dimension
- * -> [ derivative->channel = filter_size (height) ]
- * x [input_dim.channel * kernel_size[0] * kernel_size[1] (width) ]
- */
-
- int status = ML_ERROR_NONE;
- for (unsigned int b = 0; b < in_dim.batch(); ++b) {
- std::vector<float> out(kernel_size[0] * kernel_size[1] * in_dim.channel() *
- filter_size);
-
- Tensor inSub(
- TensorDim(1, in_dim.channel(), in_dim.height(), in_dim.width()),
- input.getAddress(b * input.getDim().getFeatureLen()));
-
- status = conv2d_gemm(
- derivative->getAddress(b * derivative->getDim().getFeatureLen()),
- TensorDim(1, derivative->channel(), derivative->height(),
- derivative->width()),
- inSub,
- TensorDim(1, 1, filter_size,
- kernel_size[0] * kernel_size[1] * in_dim.channel()),
- stride, padding, out.data(), out.size(), false);
- if (status != ML_ERROR_NONE)
- throw std::runtime_error("Backwarding Convolution failed.");
-
- for (unsigned int i = 0; i < filter_size; ++i) {
- Tensor &delK = weightAt(i).getGradientRef();
- Tensor &delBias = weightAt(i + filter_size).getGradientRef();
- float *del = delK.getData();
- unsigned int s = kernel_size[0] * kernel_size[1] * in_dim.channel();
-
- for (unsigned int k = 0; k < s; ++k) {
- del[k] += out[i * s + k];
- }
-
- float sum = 0.0;
- for (unsigned int j = 0; j < derivative->height(); ++j) {
- for (unsigned int k = 0; k < derivative->width(); ++k) {
- sum += derivative->getValue(b, i, j, k);
- }
- }
- delBias.setValue(0, 0, 0, 0, sum + delBias.getValue(0, 0, 0, 0));
- }
- }
-
- /** Calculate return derivative
- *
- * This is the 2D Matrix Shape [ height ] x [ width ]
- * . Height : filter.channel = input_dim.channel
- * . Width : filter_size * kernel_size[0] * kernel_size[1]
- *
- * kernel
- * f0 f1 fn
- * +---|---|---|---|...|---|---+
- * |---|---|---|---|...|---|---|
- * [filter.channel(height)] |---|---|---|---|...|---|---|
- * (=input_dim.channel) |---|---|---|---|...|---|---|
- * +---|---|---|---|...|---|---+
- * [ filter_size
- * * kernel_size[0]
- * * kernel_size[1] (width) ]
- *
- *
- * After im2Col with channel_mode true ( in : derivative with full padding )
- *
- * This is the 2D Matrix Shape [ height ] x [ width ]
- * . Height : filter_size * kernel_size[0] * kernel_size[1]
- * . Width : (input_dim.height + padding[0]*2) x (input_dim.width +
- * padding[1]*2)
- *
- * +-|-|-|-| |-|-|-|-+
- * | | | | | | | | | |
- * [ filter_size |_|_|_|_| |_|_|_|_|
- * * kernel_size[0] | | | | | .... | | | | |
- * * kernel_size[1] |_|_|_|_| |_|_|_|_|
- * (height) ] | | | | | | | | | |
- * +_|_|_|_| |_|_|_|_+
- * [(input_dim.height() + padding[0] *2)
- * * (input_dim.width() + padding[1] *2)]
- *
- * Output Dimension
- *
- * -> [ input_dim.chennel (height) ]
- * x [(input_dim.height() + padding[0]*2)
- * *(input_dim.width() + padding[1]*2) (width)]
- */
-
- TensorDim kdim(ret.channel(), filter_size, kernel_size[0], kernel_size[1]);
-
- std::vector<float> imkernel(kdim.getDataLen());
-
- unsigned int count = 0;
- float *d = imkernel.data();
-
- for (unsigned int j = 0; j < ret.channel(); ++j) {
- for (unsigned int i = 0; i < filter_size; ++i) {
- Tensor &filters = weightAt(i).getVariableRef();
- for (unsigned int k = 0; k < kernel_size[0] * kernel_size[1]; ++k) {
- d[count++] = filters.getData()[j * kernel_size[0] * kernel_size[1] + k];
- }
- }
- }
-
- TensorDim input_dim_padded(1, in_dim.channel(),
- in_dim.height() + padding[0] * 2,
- in_dim.width() + padding[1] * 2);
-
- for (unsigned int b = 0; b < in_dim.batch(); ++b) {
- Tensor inSub(
- TensorDim(1, derivative->channel(), derivative->height(),
- derivative->width()),
- derivative->getAddress(b * derivative->getDim().getFeatureLen()));
-
- status =
- conv2d_gemm(imkernel.data(), kdim, inSub, input_dim_padded, stride,
- same_pad, ret.getAddress(b * ret.getDim().getFeatureLen()),
- input_dim_padded.getFeatureLen(), true);
- if (status != ML_ERROR_NONE)
- throw std::runtime_error("Backwarding Convolution failed.");
- }
-
- if (trainable) {
- // Update K / bias
- for (unsigned int i = 0; i < filter_size; ++i) {
- Tensor &delK = weightAt(i).getGradientRef();
- Tensor &filters = weightAt(i).getVariableRef();
-
- if (isWeightRegularizerL2Norm()) {
- status = delK.add_i(filters, weight_regularizer_constant);
- if (status != ML_ERROR_NONE)
- throw std::runtime_error("Weight regularization failed");
- }
- }
-
- opt->apply_gradients(weight_list, num_weights, iteration);
- }
-
- return {MAKE_SHARED_TENSOR(std::move(strip_pad(ret, padding.data())))};
-}
-
-void Conv2DLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<Conv2DLayer> from = std::static_pointer_cast<Conv2DLayer>(l);
- this->filter_size = from->filter_size;
- for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
- this->kernel_size[i] = from->kernel_size[i];
- this->stride[i] = from->stride[i];
- this->padding[i] = from->padding[i];
- }
-}
-
-int Conv2DLayer::setSize(int *size, PropertyType type) {
- int status = ML_ERROR_NONE;
- switch (type) {
- case PropertyType::kernel_size:
- for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
- kernel_size[i] = size[i];
- }
- break;
- case PropertyType::stride:
- for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
- stride[i] = size[i];
- }
- break;
- case PropertyType::padding:
- for (unsigned int i = 0; i < CONV2D_DIM; ++i) {
- padding[i] = size[i];
- }
- break;
- default:
- ml_loge("Error: Unknown Layer Property type");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
- return status;
-}
-
-int Conv2DLayer::setFilter(int f) {
- int status = ML_ERROR_NONE;
- if (f <= 0) {
- ml_loge("Error: number of filters must be greater than 0");
- status = ML_ERROR_INVALID_PARAMETER;
- }
- filter_size = f;
- return status;
-}
-
-void Conv2DLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::filters: {
- if (!value.empty()) {
- status = setUint(filter_size, value);
- throw_status(status);
- }
- } break;
- case PropertyType::kernel_size:
- if (!value.empty()) {
- status = getValues(CONV2D_DIM, value, (int *)(kernel_size.data()));
- throw_status(status);
- if (kernel_size[0] == 0 || kernel_size[1] == 0) {
- throw std::invalid_argument(
- "[Conv2DLayer] kernel_size must be greater than 0");
- }
- }
- break;
- case PropertyType::stride:
- if (!value.empty()) {
- status = getValues(CONV2D_DIM, value, (int *)(stride.data()));
- throw_status(status);
- if (stride[0] == 0 || stride[1] == 0) {
- throw std::invalid_argument(
- "[Conv2DLayer] stride must be greater than 0");
- }
- }
- break;
- case PropertyType::padding:
- if (!value.empty()) {
- status = getValues(CONV2D_DIM, value, (int *)(padding.data()));
- throw_status(status);
- }
- break;
- case PropertyType::normalization:
- if (!value.empty()) {
- status = setBoolean(normalization, value);
- throw_status(status);
- }
- break;
- case PropertyType::standardization:
- if (!value.empty()) {
- status = setBoolean(standardization, value);
- throw_status(status);
- }
- break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-int Conv2DLayer::conv2d(float *in, TensorDim indim, const float *kernel,
- TensorDim kdim, float *out, unsigned int const *stride,
- float bias) {
-
- int status = ML_ERROR_NONE;
- unsigned int channel = indim.channel();
- unsigned int height = indim.height();
- unsigned int width = indim.width();
- unsigned int k_width = kdim.width();
- unsigned int k_height = kdim.height();
- unsigned int o_width = ((indim.width() - kdim.width()) / stride[0] + 1);
-
- if (indim.channel() != kdim.channel()) {
- ml_loge("Error: Input and Kenel Dimension is not match!");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- // Optimizer This routine : There are lots of dulicated calculations
- unsigned int I = 0;
- unsigned int J = 0;
- for (unsigned int j = 0; j < height - k_height + 1; j += stride[0]) {
- J = 0;
- for (unsigned int k = 0; k < width - k_width + 1; k += stride[1]) {
- float sum = 0.0f;
- for (unsigned int i = 0; i < channel; ++i) {
- for (unsigned int ki = 0; ki < k_height; ++ki) {
- for (unsigned int kj = 0; kj < k_width; ++kj) {
- sum += kernel[i * k_height * k_width + ki * k_width + kj] *
- in[i * height * width + (j + ki) * width + (k + kj)];
- }
- }
- }
- sum += bias;
- out[I * o_width + J] = sum;
- J++;
- }
- I++;
- }
- return status;
-}
-
-int Conv2DLayer::conv2d_gemm(
- const float *mkernel, TensorDim kdim, Tensor const &in, TensorDim outdim,
- const std::array<unsigned int, CONV2D_DIM> &mstride,
- const std::array<unsigned int, CONV2D_DIM> &pad, float *out,
- unsigned int osize, bool channel_mode) {
- int status = ML_ERROR_NONE;
- std::vector<float> in_col;
-
- if (channel_mode) {
- in_col.resize(kdim.getFeatureLen() * outdim.width() * outdim.height());
- } else {
- in_col.resize(kdim.width() * kdim.height() * outdim.width());
- }
-
- Tensor in_padded = zero_pad(0, in, pad.data());
- status =
- im2col(in_padded, kdim, in_col.data(), outdim, mstride, channel_mode);
- if (status != ML_ERROR_NONE)
- throw std::runtime_error("Forwarding Convolution failed.");
-
- float alpha_dgemm = 1.0f;
- float beta_dgemm = 0.0f;
- const float *data = mkernel;
- const float *mdata = in_col.data();
- float *rdata = out;
-
- unsigned int kh, kw, w;
-
- if (channel_mode) {
- kh = kdim.batch();
- kw = kdim.getFeatureLen();
- w = outdim.width() * outdim.height();
- } else {
- kh = outdim.height();
- kw = kdim.width() * kdim.height();
- w = outdim.width();
- }
-
- sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, kh, w, kw, alpha_dgemm, data,
- kw, mdata, w, beta_dgemm, rdata, w);
-
- return status;
-}
-
-int Conv2DLayer::im2col(Tensor in_padded, TensorDim kdim, float *in_col,
- TensorDim outdim,
- const std::array<unsigned int, CONV2D_DIM> &mstride,
- bool channel_mode) {
-
- int status = ML_ERROR_NONE;
- unsigned int count;
- unsigned int channel = in_padded.channel();
- unsigned int height = in_padded.height();
- unsigned int width = in_padded.width();
- unsigned int k_width = kdim.width();
- unsigned int k_height = kdim.height();
-
- unsigned int J = 0;
- if (channel_mode) {
- for (unsigned int j = 0; j <= height - k_height; j += mstride[0]) {
- for (unsigned int k = 0; k <= width - k_width; k += mstride[1]) {
- count = 0;
- for (unsigned int i = 0; i < channel; ++i) {
- for (unsigned int ki = 0; ki < k_height; ++ki) {
- for (unsigned int kj = 0; kj < k_width; ++kj) {
- in_col[count * (outdim.width() * outdim.height()) + J] =
- in_padded
- .getData()[i * height * width + (j + ki) * width + (k + kj)];
- count++;
- }
- }
- }
- J++;
- }
- }
- if (J != outdim.width() * outdim.height())
- status = ML_ERROR_INVALID_PARAMETER;
- } else {
- for (unsigned int i = 0; i < channel; ++i) {
- for (unsigned int j = 0; j <= height - k_height; j += mstride[0]) {
- for (unsigned int k = 0; k <= width - k_width; k += mstride[1]) {
- count = 0;
- for (unsigned int ki = 0; ki < k_height; ++ki) {
- for (unsigned int kj = 0; kj < k_width; ++kj) {
- in_col[count * (outdim.width()) + J] =
- in_padded
- .getData()[i * height * width + (j + ki) * width + (k + kj)];
- count++;
- }
- }
- J++;
- }
- }
- }
- if (J != outdim.width())
- status = ML_ERROR_INVALID_PARAMETER;
- }
-
- return status;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer.cpp
- * @date 04 December 2019
- * @brief This is buffer object to handle big data
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <cassert>
-#include <climits>
-#include <condition_variable>
-#include <cstring>
-#include <databuffer.h>
-#include <databuffer_util.h>
-#include <functional>
-#include <iomanip>
-#include <mutex>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <sstream>
-#include <stdexcept>
-#include <stdio.h>
-#include <stdlib.h>
-#include <thread>
-#include <util_func.h>
-
-std::exception_ptr globalExceptionPtr = nullptr;
-
-namespace nntrainer {
-
-constexpr char USER_DATA[] = "user_data";
-
-std::mutex data_lock;
-
-std::mutex readyTrainData;
-std::mutex readyValData;
-std::mutex readyTestData;
-
-std::condition_variable cv_train;
-std::condition_variable cv_val;
-std::condition_variable cv_test;
-
-DataBuffer::DataBuffer(DataBufferType type) :
- train_running(),
- val_running(),
- test_running(),
- train_thread(),
- val_thread(),
- test_thread(),
- data_buffer_type(type),
- user_data(nullptr) {
- SET_VALIDATION(false);
- class_num = 0;
- cur_train_bufsize = 0;
- cur_val_bufsize = 0;
- cur_test_bufsize = 0;
- train_bufsize = 0;
- val_bufsize = 0;
- test_bufsize = 0;
- max_train = 0;
- max_val = 0;
- max_test = 0;
- rest_train = 0;
- rest_val = 0;
- rest_test = 0;
- batch_size = 0;
- train_running = false;
- val_running = false;
- test_running = false;
- trainReadyFlag = DATA_NOT_READY;
- valReadyFlag = DATA_NOT_READY;
- testReadyFlag = DATA_NOT_READY;
- rng.seed(getSeed());
-};
-
-int DataBuffer::rangeRandom(int min, int max) {
- std::uniform_int_distribution<int> dist(min, max);
- return dist(rng);
-}
-
-int DataBuffer::run(BufferType type) {
- int status = ML_ERROR_NONE;
- switch (type) {
- case BufferType::BUF_TRAIN:
- if (trainReadyFlag == DATA_ERROR)
- return ML_ERROR_INVALID_PARAMETER;
-
- if (validation[DATA_TRAIN]) {
- this->train_running = true;
- this->train_thread = std::thread(&DataBuffer::updateData, this, type);
- if (globalExceptionPtr) {
- try {
- std::rethrow_exception(globalExceptionPtr);
- } catch (const std::exception &ex) {
- std::cout << ex.what() << "\n";
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
- } else {
- ml_loge("Error: Training Data Set is not valid");
- return ML_ERROR_INVALID_PARAMETER;
- }
- break;
- case BufferType::BUF_VAL:
- if (valReadyFlag == DATA_ERROR)
- return ML_ERROR_INVALID_PARAMETER;
- if (validation[DATA_VAL]) {
- this->val_running = true;
- this->val_thread = std::thread(&DataBuffer::updateData, this, type);
- if (globalExceptionPtr) {
- try {
- std::rethrow_exception(globalExceptionPtr);
- } catch (const std::exception &ex) {
- std::cout << ex.what() << "\n";
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
- } else {
- ml_loge("Error: Validation Data Set is not valid");
- return ML_ERROR_INVALID_PARAMETER;
- }
- break;
- case BufferType::BUF_TEST:
- if (testReadyFlag == DATA_ERROR)
- return ML_ERROR_INVALID_PARAMETER;
-
- if (validation[DATA_TEST]) {
- this->test_running = true;
- this->test_thread = std::thread(&DataBuffer::updateData, this, type);
- if (globalExceptionPtr) {
- try {
- std::rethrow_exception(globalExceptionPtr);
- } catch (const std::exception &ex) {
- std::cout << ex.what() << "\n";
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
- } else {
- ml_loge("Error: Test Data Set is not valid");
- return ML_ERROR_INVALID_PARAMETER;
- }
- break;
- default:
- ml_loge("Error: Not Supported Data Type");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
-
- return status;
-}
-
-int DataBuffer::clear(BufferType type) {
- int status = ML_ERROR_NONE;
- NN_EXCEPTION_NOTI(DATA_NOT_READY);
- switch (type) {
- case BufferType::BUF_TRAIN: {
- train_running = false;
- if (validation[DATA_TRAIN] && true == train_thread.joinable())
- train_thread.join();
- this->train_data.clear();
- this->train_data_label.clear();
- this->cur_train_bufsize = 0;
- this->rest_train = max_train;
- } break;
- case BufferType::BUF_VAL: {
- val_running = false;
- if (validation[DATA_VAL] && true == val_thread.joinable())
- val_thread.join();
- this->val_data.clear();
- this->val_data_label.clear();
- this->cur_val_bufsize = 0;
- this->rest_val = max_val;
- } break;
- case BufferType::BUF_TEST: {
- test_running = false;
- if (validation[DATA_TEST] && true == test_thread.joinable())
- test_thread.join();
- this->test_data.clear();
- this->test_data_label.clear();
- this->cur_test_bufsize = 0;
- this->rest_test = max_test;
- } break;
- default:
- ml_loge("Error: Not Supported Data Type");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
- return status;
-}
-
-int DataBuffer::clear() {
- unsigned int i;
-
- int status = ML_ERROR_NONE;
- for (i = (int)BufferType::BUF_TRAIN; i <= (int)BufferType::BUF_TEST; ++i) {
- BufferType type = static_cast<BufferType>(i);
- status = this->clear(type);
-
- if (status != ML_ERROR_NONE) {
- ml_loge("Error: error occurred during clearing");
- return status;
- }
- }
-
- return status;
-}
-
-bool DataBuffer::getDataFromBuffer(BufferType type, float *out, float *label) {
-
- using QueueType = std::vector<std::vector<float>>;
-
- auto wait_for_data_fill = [](std::mutex &ready_mutex,
- std::condition_variable &cv, DataStatus &flag,
- const unsigned int batch_size,
- QueueType &queue) {
- while (true) {
- std::unique_lock<std::mutex> ul(ready_mutex);
- cv.wait(ul, [&]() -> bool { return flag; });
- if (flag == DATA_ERROR || flag == DATA_END)
- return queue.size() < batch_size ? false : true;
-
- if (flag == DATA_READY && queue.size() >= batch_size)
- return true;
- }
-
- throw std::logic_error("[getDataFromBuffer] control should not reach here");
- };
-
- auto fill_bundled_data_from_queue =
- [](std::mutex &q_lock, QueueType &q, const unsigned int batch_size,
- const unsigned int feature_size, float *buf) {
- for (unsigned int b = 0; b < batch_size; ++b)
- std::copy(q[b].begin(), q[b].begin() + feature_size,
- buf + b * feature_size);
-
- q_lock.lock();
- q.erase(q.begin(), q.begin() + batch_size);
- q_lock.unlock();
- };
-
- /// facade that wait for the databuffer to be filled and pass it to outparam
- /// note that batch_size is passed as an argument because it can vary by
- /// BufferType::BUF_TYPE later...
- auto fill_out_params =
- [&](std::mutex &ready_mutex, std::condition_variable &cv, DataStatus &flag,
- QueueType &data_q, QueueType &label_q, const unsigned int batch_size,
- unsigned int &cur_bufsize) {
- if (!wait_for_data_fill(ready_mutex, cv, flag, batch_size, data_q)) {
- return false;
- }
-
- fill_bundled_data_from_queue(data_lock, data_q, batch_size,
- this->input_dim.getFeatureLen(), out);
- fill_bundled_data_from_queue(data_lock, label_q, batch_size,
- this->class_num, label);
-
- cur_bufsize -= batch_size;
- return true;
- };
-
- switch (type) {
- case BufferType::BUF_TRAIN:
- if (!fill_out_params(readyTrainData, cv_train, trainReadyFlag, train_data,
- train_data_label, batch_size, cur_train_bufsize))
- return false;
- break;
- case BufferType::BUF_VAL:
- if (!fill_out_params(readyValData, cv_val, valReadyFlag, val_data,
- val_data_label, batch_size, cur_val_bufsize))
- return false;
- break;
- case BufferType::BUF_TEST:
- if (!fill_out_params(readyTestData, cv_test, testReadyFlag, test_data,
- test_data_label, batch_size, cur_test_bufsize))
- return false;
- break;
- default:
- ml_loge("Error: Not Supported Data Type");
- return false;
- break;
- }
-
- return true;
-}
-
-int DataBuffer::setClassNum(unsigned int num) {
- int status = ML_ERROR_NONE;
- if (num <= 0) {
- ml_loge("Error: number of class should be bigger than 0");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
- if (class_num != 0 && class_num != num) {
- ml_loge("Error: number of class should be same with number of label label");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
- class_num = num;
- return status;
-}
-
-int DataBuffer::setBufSize(unsigned int size) {
- int status = ML_ERROR_NONE;
- train_bufsize = size;
- val_bufsize = size;
- test_bufsize = size;
- return status;
-}
-
-int DataBuffer::setBatchSize(unsigned int size) {
- int status = ML_ERROR_NONE;
- if (size == 0) {
- ml_loge("Error: batch size must be greater than 0");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
- batch_size = size;
- return status;
-}
-
-int DataBuffer::init() {
- if (batch_size == 0) {
- ml_loge("Error: batch size must be greater than 0");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- /** for now, train_bufsize, val_bufsize and test_bufsize are same value */
- if (train_bufsize < batch_size) {
- if (train_bufsize > 1) {
- ml_logw("Dataset buffer size reset to be at least batch size");
- }
- train_bufsize = batch_size;
- val_bufsize = batch_size;
- test_bufsize = batch_size;
- }
-
- if (!class_num) {
- ml_loge("Error: number of class must be set");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- if (!this->input_dim.getFeatureLen()) {
- ml_loge("Error: feature size must be set");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- this->cur_train_bufsize = 0;
- this->cur_val_bufsize = 0;
- this->cur_test_bufsize = 0;
-
- readyTrainData.lock();
- trainReadyFlag = DATA_NOT_READY;
- readyTrainData.unlock();
- readyValData.lock();
- valReadyFlag = DATA_NOT_READY;
- readyValData.unlock();
- readyTestData.lock();
- testReadyFlag = DATA_NOT_READY;
- readyTestData.unlock();
- return ML_ERROR_NONE;
-}
-
-int DataBuffer::setFeatureSize(TensorDim indim) {
- int status = ML_ERROR_NONE;
- input_dim = indim;
- return status;
-}
-
-void DataBuffer::displayProgress(const int count, BufferType type, float loss) {
- int barWidth = 20;
- float max_size = max_train;
- switch (type) {
- case BufferType::BUF_TRAIN:
- max_size = max_train;
- break;
- case BufferType::BUF_VAL:
- max_size = max_val;
- break;
- case BufferType::BUF_TEST:
- max_size = max_test;
- break;
- default:
- ml_loge("Error: Not Supported Data Type");
- break;
- }
- std::stringstream ssInt;
- ssInt << count * batch_size;
-
- std::string str = ssInt.str();
- int len = str.length();
-
- if (max_size == 0) {
- int pad_left = (barWidth - len) / 2;
- int pad_right = barWidth - pad_left - len;
- std::string out_str =
- std::string(pad_left, ' ') + str + std::string(pad_right, ' ');
- std::cout << " [ ";
- std::cout << out_str;
- std::cout << " ] "
- << " ( Training Loss: " << loss << " )\r";
- } else {
- float progress;
- if (batch_size > max_size)
- progress = 1.0;
- else
- progress = (((float)(count * batch_size)) / max_size);
-
- int pos = barWidth * progress;
- std::cout << " [ ";
- for (int l = 0; l < barWidth; ++l) {
- if (l <= pos)
- std::cout << "=";
- else
- std::cout << " ";
- }
- std::cout << " ] " << int(progress * 100.0) << "% ( Training Loss: " << loss
- << " )\r";
- }
-
- std::cout.flush();
-}
-
-int DataBuffer::setProperty(std::vector<void *> values) {
- int status = ML_ERROR_NONE;
- std::vector<std::string> properties;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- char *key_ptr = (char *)values[i];
- std::string key = key_ptr;
- std::string value;
-
- /** Handle the user_data as a special case */
- if (key == USER_DATA) {
- /** This ensures that a valid user_data element is passed by the user */
- if (i + 1 >= values.size())
- return ML_ERROR_INVALID_PARAMETER;
-
- this->user_data = values[i + 1];
-
- /** As values of i+1 is consumed, increase i by 1 */
- i++;
- } else {
- properties.push_back(key);
- continue;
- }
- }
-
- status = setProperty(properties);
-
- return status;
-}
-
-int DataBuffer::setProperty(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
- status = getKeyValue(values[i], key, value);
- NN_RETURN_STATUS();
-
- unsigned int type = parseDataProperty(key);
- if (value.empty())
- return ML_ERROR_INVALID_PARAMETER;
-
- status = setProperty(static_cast<PropertyType>(type), value);
- NN_RETURN_STATUS();
- }
-
- return status;
-}
-
-int DataBuffer::setProperty(const PropertyType type, std::string &value) {
- int status = ML_ERROR_NONE;
- unsigned int size = 0;
-
- switch (type) {
- case PropertyType::buffer_size:
- status = setUint(size, value);
- NN_RETURN_STATUS();
- status = this->setBufSize(size);
- NN_RETURN_STATUS();
- break;
- default:
- ml_loge("Error: Unknown Data Buffer Property Key");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
-
- return status;
-}
-
-int DataBuffer::setGeneratorFunc(BufferType type, datagen_cb func) {
- return ML_ERROR_NOT_SUPPORTED;
-}
-
-int DataBuffer::setDataFile(DataType type, std::string path) {
- return ML_ERROR_NOT_SUPPORTED;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file optimizer_factory.cpp
- * @date 11 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the databuffer factory.
- */
-
-#include <databuffer_factory.h>
-
-#include <databuffer_file.h>
-#include <databuffer_func.h>
-#include <nntrainer_error.h>
-
-namespace nntrainer {
-
-/**
- * @brief Factory creator with constructor
- */
-std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type) {
- switch (type) {
- case DataBufferType::GENERATOR:
- return std::make_unique<DataBufferFromCallback>();
- case DataBufferType::FILE:
- return std::make_unique<DataBufferFromDataFile>();
- case DataBufferType::UNKNOWN:
- /** fallthrough intended */
- default:
- throw std::invalid_argument("Unknown type for the dataset");
- }
-}
-
-/**
- * @brief Factory creator with constructor for dataset
- */
-std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
- const char *train_file,
- const char *valid_file,
- const char *test_file) {
- if (type != DataBufferType::FILE)
- throw std::invalid_argument(
- "Cannot create dataset with files with the given dataset type");
-
- std::unique_ptr<DataBuffer> dataset = createDataBuffer(type);
-
- if (dataset->setDataFile(DataType::DATA_TRAIN, train_file) != ML_ERROR_NONE)
- throw std::invalid_argument("Invalid train file");
-
- if (valid_file &&
- dataset->setDataFile(DataType::DATA_VAL, valid_file) != ML_ERROR_NONE)
- throw std::invalid_argument("Invalid valid file");
-
- if (test_file &&
- dataset->setDataFile(DataType::DATA_TEST, test_file) != ML_ERROR_NONE)
- throw std::invalid_argument("Invalid test file");
-
- return dataset;
-}
-
-/**
- * @brief Factory creator with constructor for dataset
- */
-std::unique_ptr<DataBuffer> createDataBuffer(DataBufferType type,
- datagen_cb train, datagen_cb valid,
- datagen_cb test) {
- if (type != DataBufferType::GENERATOR)
- throw std::invalid_argument("Cannot create dataset with generator "
- "callbacks with the given dataset type");
-
- std::unique_ptr<DataBuffer> dataset = createDataBuffer(type);
-
- if (dataset->setGeneratorFunc((BufferType)DataType::DATA_TRAIN, train) !=
- ML_ERROR_NONE)
- throw std::invalid_argument("Invalid train data generator");
-
- if (valid && dataset->setGeneratorFunc((BufferType)DataType::DATA_VAL,
- valid) != ML_ERROR_NONE)
- throw std::invalid_argument("Invalid valid data generator");
-
- if (test && dataset->setGeneratorFunc((BufferType)DataType::DATA_TEST,
- test) != ML_ERROR_NONE)
- throw std::invalid_argument("Invalid test data generator");
-
- return dataset;
-}
-
-} // namespace nntrainer
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.cpp
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <assert.h>
-#include <climits>
-#include <condition_variable>
-#include <cstring>
-#include <databuffer_file.h>
-#include <databuffer_util.h>
-#include <functional>
-#include <iomanip>
-#include <mutex>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <sstream>
-#include <stdexcept>
-#include <stdio.h>
-#include <stdlib.h>
-#include <thread>
-
-extern std::exception_ptr globalExceptionPtr;
-
-namespace nntrainer {
-
-extern std::mutex data_lock;
-
-extern std::mutex readyTrainData;
-extern std::mutex readyValData;
-extern std::mutex readyTestData;
-
-extern std::condition_variable cv_train;
-extern std::condition_variable cv_val;
-extern std::condition_variable cv_test;
-
-static long getFileSize(std::string file_name) {
- std::ifstream file_stream(file_name.c_str(), std::ios::in | std::ios::binary);
- if (file_stream.good()) {
- file_stream.seekg(0, std::ios::end);
- return file_stream.tellg();
- } else {
- return 0;
- }
-}
-
-int DataBufferFromDataFile::init() {
- int status = ML_ERROR_NONE;
-
- status = DataBuffer::init();
- if (status != ML_ERROR_NONE)
- return status;
-
- if (validation[DATA_TRAIN] && max_train < batch_size) {
- max_train = batch_size;
- }
-
- if (validation[DATA_VAL] && max_val < batch_size) {
- max_val = batch_size;
- }
-
- if (validation[DATA_TEST] && max_test < batch_size) {
- max_test = batch_size;
- }
-
- this->rest_train = max_train;
- this->rest_val = max_val;
- this->rest_test = max_test;
-
- this->train_running = true;
- this->val_running = true;
- this->test_running = true;
-
- if (validation[DATA_TRAIN] && max_train < train_bufsize) {
- ml_logw("Warning: Total number of train is less than train buffer size. "
- "Train buffer size is set as total number of train");
- train_bufsize = batch_size;
- }
-
- if (validation[DATA_VAL] && max_val < val_bufsize) {
- ml_logw(
- "Warning: Total number of validation is less than validation buffer "
- "size. Validation buffer size is set as total number of validation");
- val_bufsize = batch_size;
- }
-
- if (validation[DATA_TEST] && max_test < test_bufsize) {
- ml_logw("Warning: Total number of test is less than test buffer size. Test "
- "buffer size is set as total number of test");
- test_bufsize = batch_size;
- }
-
- return ML_ERROR_NONE;
-}
-
-void DataBufferFromDataFile::updateData(BufferType type) {
- unsigned int max_size = 0;
- unsigned int buf_size = 0;
- unsigned int *rest_size = NULL;
- unsigned int *cur_size = NULL;
- bool *running = NULL;
- std::vector<std::vector<float>> *data = NULL;
- std::vector<std::vector<float>> *datalabel = NULL;
- std::ifstream file;
- switch (type) {
- case BufferType::BUF_TRAIN: {
- max_size = max_train;
- buf_size = train_bufsize;
- rest_size = &rest_train;
- cur_size = &cur_train_bufsize;
- running = &train_running;
- data = &train_data;
- datalabel = &train_data_label;
-
- std::ifstream train_stream(train_name, std::ios::in | std::ios::binary);
- file.swap(train_stream);
- readyTrainData.lock();
- trainReadyFlag = DATA_NOT_READY;
- readyTrainData.unlock();
-
- } break;
- case BufferType::BUF_VAL: {
- max_size = max_val;
- buf_size = val_bufsize;
- rest_size = &rest_val;
- cur_size = &cur_val_bufsize;
- running = &val_running;
- data = &val_data;
- datalabel = &val_data_label;
-
- std::ifstream val_stream(val_name, std::ios::in | std::ios::binary);
- file.swap(val_stream);
- readyValData.lock();
- valReadyFlag = DATA_NOT_READY;
- readyValData.unlock();
-
- } break;
- case BufferType::BUF_TEST: {
- max_size = max_test;
- buf_size = test_bufsize;
- rest_size = &rest_test;
- cur_size = &cur_test_bufsize;
- running = &test_running;
- data = &test_data;
- datalabel = &test_data_label;
-
- std::ifstream test_stream(test_name, std::ios::in | std::ios::binary);
- file.swap(test_stream);
- readyTestData.lock();
- testReadyFlag = DATA_NOT_READY;
- readyTestData.unlock();
- } break;
- default:
- try {
- throw std::runtime_error("Error: Not Supported Data Type");
- } catch (...) {
- globalExceptionPtr = std::current_exception();
- NN_EXCEPTION_NOTI(DATA_ERROR);
- return;
- }
- break;
- }
-
- unsigned int I;
- std::vector<unsigned int> mark;
- mark.resize(max_size);
- file.clear();
- file.seekg(0, std::ios_base::end);
- uint64_t file_length = file.tellg();
-
- for (unsigned int i = 0; i < max_size; ++i) {
- mark[i] = i;
- }
-
- while ((*running)) {
-
- if (mark.size() == 0) {
- NN_EXCEPTION_NOTI(DATA_END);
- break;
- }
-
- if (buf_size - (*cur_size) > 0 && (*rest_size) > 0) {
- std::vector<float> vec;
- std::vector<float> veclabel;
-
- unsigned int id = rangeRandom(0, mark.size() - 1);
- I = mark[id];
-
- try {
- if (I > max_size) {
- throw std::runtime_error(
- "Error: Test case id cannot exceed maximum number of test");
- }
- } catch (...) {
- globalExceptionPtr = std::current_exception();
- NN_EXCEPTION_NOTI(DATA_ERROR);
- return;
- }
-
- mark.erase(mark.begin() + id);
-
- uint64_t position =
- (uint64_t)((I * input_dim.getFeatureLen() + (uint64_t)I * class_num) *
- sizeof(float));
- try {
- if (position > file_length) {
- throw std::runtime_error("Error: Cannot exceed max file size");
- }
- } catch (...) {
- globalExceptionPtr = std::current_exception();
- NN_EXCEPTION_NOTI(DATA_ERROR);
- return;
- }
-
- file.seekg(position, std::ios::beg);
-
- for (unsigned int j = 0; j < input_dim.getFeatureLen(); ++j) {
- float d;
- file.read((char *)&d, sizeof(float));
- vec.push_back(d);
- }
-
- for (unsigned int j = 0; j < class_num; ++j) {
- float d;
- file.read((char *)&d, sizeof(float));
- veclabel.push_back(d);
- }
-
- data_lock.lock();
- data->push_back(vec);
- datalabel->push_back(veclabel);
- (*rest_size)--;
- (*cur_size)++;
- data_lock.unlock();
- }
- if (buf_size == (*cur_size)) {
- NN_EXCEPTION_NOTI(DATA_READY);
- }
- }
-
- file.close();
-}
-
-int DataBufferFromDataFile::setDataFile(DataType type, std::string path) {
- int status = ML_ERROR_NONE;
- std::ifstream data_file(path.c_str());
-
- switch (type) {
- case DATA_TRAIN: {
- validation[type] = true;
- if (!data_file.good()) {
- ml_loge(
- "Error: Cannot open data file, Datafile is necessary for training");
- validation[type] = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
- train_name = path;
- } break;
- case DATA_VAL: {
- validation[type] = true;
- if (!data_file.good()) {
- ml_loge("Error: Cannot open validation data file. Cannot validate "
- "training result");
- validation[type] = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
- val_name = path;
- } break;
- case DATA_TEST: {
- validation[type] = true;
- if (!data_file.good()) {
- ml_loge("Error: Cannot open test data file. Cannot test training result");
- validation[type] = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
- test_name = path;
- } break;
- case DATA_LABEL: {
- std::string data;
- validation[type] = true;
- if (!data_file.good()) {
- ml_loge("Error: Cannot open label file");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
- while (data_file >> data) {
- labels.push_back(data);
- }
- if (class_num != 0 && class_num != labels.size()) {
- ml_loge("Error: number of label should be same with number class number");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- }
- class_num = labels.size();
- } break;
- case DATA_UNKNOWN:
- default:
- ml_loge("Error: Not Supported Data Type");
- SET_VALIDATION(false);
- return ML_ERROR_INVALID_PARAMETER;
- break;
- }
- return status;
-}
-
-int DataBufferFromDataFile::setFeatureSize(TensorDim tdim) {
- int status = ML_ERROR_NONE;
- long file_size = 0;
-
- status = DataBuffer::setFeatureSize(tdim);
- if (status != ML_ERROR_NONE)
- return status;
-
- if (validation[DATA_TRAIN]) {
- file_size = getFileSize(train_name);
- max_train = static_cast<unsigned int>(
- file_size /
- (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
- if (max_train < batch_size) {
- ml_logw("Warning: number of training data is smaller than batch size");
- }
- } else {
- max_train = 0;
- }
-
- if (validation[DATA_VAL]) {
- file_size = getFileSize(val_name);
- max_val = static_cast<unsigned int>(
- file_size /
- (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
- if (max_val < batch_size) {
- ml_logw("Warning: number of val data is smaller than batch size");
- }
- } else {
- max_val = 0;
- }
-
- if (validation[DATA_TEST]) {
- file_size = getFileSize(test_name);
- max_test = static_cast<unsigned int>(
- file_size /
- (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
- if (max_test < batch_size) {
- ml_logw("Warning: number of test data is smaller than batch size");
- }
- } else {
- max_test = 0;
- }
- return status;
-}
-
-int DataBufferFromDataFile::setProperty(const PropertyType type,
- std::string &value) {
- int status = ML_ERROR_NONE;
-
- if (data_buffer_type != DataBufferType::FILE)
- return ML_ERROR_INVALID_PARAMETER;
-
- switch (type) {
- case PropertyType::train_data:
- status = this->setDataFile(DATA_TRAIN, value);
- break;
- case PropertyType::val_data:
- status = this->setDataFile(DATA_VAL, value);
- break;
- case PropertyType::test_data:
- status = this->setDataFile(DATA_TEST, value);
- break;
- case PropertyType::label_data:
- status = this->setDataFile(DATA_LABEL, value);
- break;
- default:
- status = DataBuffer::setProperty(type, value);
- break;
- }
-
- return status;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.cpp
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <assert.h>
-#include <climits>
-#include <condition_variable>
-#include <cstring>
-#include <databuffer_func.h>
-#include <databuffer_util.h>
-#include <functional>
-#include <iomanip>
-#include <mutex>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <sstream>
-#include <stdexcept>
-#include <stdio.h>
-#include <stdlib.h>
-#include <thread>
-
-extern std::exception_ptr globalExceptionPtr;
-
-namespace nntrainer {
-
-extern std::mutex data_lock;
-
-extern std::mutex readyTrainData;
-extern std::mutex readyValData;
-extern std::mutex readyTestData;
-
-extern std::condition_variable cv_train;
-extern std::condition_variable cv_val;
-extern std::condition_variable cv_test;
-
-int DataBufferFromCallback::init() {
- int status = ML_ERROR_NONE;
-
- status = DataBuffer::init();
- if (status != ML_ERROR_NONE)
- return status;
-
- if (callback_train == nullptr)
- return ML_ERROR_BAD_ADDRESS;
-
- this->max_train = 0;
- this->max_val = 0;
- this->max_test = 0;
-
- if (train_bufsize > batch_size || train_bufsize == 0) {
- train_bufsize = batch_size;
- }
-
- if (val_bufsize > batch_size || val_bufsize == 0) {
- val_bufsize = batch_size;
- }
-
- if (test_bufsize > batch_size || test_bufsize == 0) {
- test_bufsize = batch_size;
- }
-
- this->train_running = true;
- this->val_running = true;
- this->test_running = true;
-
- return ML_ERROR_NONE;
-}
-
-int DataBufferFromCallback::setGeneratorFunc(BufferType type, datagen_cb func) {
-
- int status = ML_ERROR_NONE;
- switch (type) {
- case BufferType::BUF_TRAIN:
- if (!func)
- return ML_ERROR_INVALID_PARAMETER;
- callback_train = func;
- if (func)
- validation[0] = true;
- break;
- case BufferType::BUF_VAL:
- callback_val = func;
- if (func)
- validation[1] = true;
- break;
- case BufferType::BUF_TEST:
- callback_test = func;
- if (func)
- validation[2] = true;
- break;
- default:
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
-
- return status;
-}
-
-void DataBufferFromCallback::updateData(BufferType type) {
- int status = ML_ERROR_NONE;
-
- unsigned int buf_size = 0;
- unsigned int *cur_size = NULL;
- bool *running = NULL;
- std::vector<std::vector<float>> *data = NULL;
- std::vector<std::vector<float>> *datalabel = NULL;
- datagen_cb callback;
-
- switch (type) {
- case BufferType::BUF_TRAIN: {
- buf_size = train_bufsize;
- cur_size = &cur_train_bufsize;
- running = &train_running;
- data = &train_data;
- datalabel = &train_data_label;
- callback = callback_train;
- } break;
- case BufferType::BUF_VAL: {
- buf_size = val_bufsize;
- cur_size = &cur_val_bufsize;
- running = &val_running;
- data = &val_data;
- datalabel = &val_data_label;
- callback = callback_val;
- } break;
- case BufferType::BUF_TEST: {
- buf_size = test_bufsize;
- cur_size = &cur_test_bufsize;
- running = &test_running;
- data = &test_data;
- datalabel = &test_data_label;
- callback = callback_test;
- } break;
- default:
- break;
- }
-
- try {
- if ((cur_size == NULL) || (running == NULL) || (data == NULL) ||
- (datalabel == NULL))
- throw std::runtime_error("Error: assining error");
- } catch (...) {
- globalExceptionPtr = std::current_exception();
- NN_EXCEPTION_NOTI(DATA_ERROR);
- return;
- }
- bool endflag = false;
-
- float **vec_arr = (float **)malloc(sizeof(float *) * 1);
- float **veclabel_arr = (float **)malloc(sizeof(float *) * 1);
-
- float *vec =
- (float *)malloc(sizeof(float) * input_dim.batch() * input_dim.channel() *
- input_dim.height() * input_dim.width());
- float *veclabel =
- (float *)malloc(sizeof(float) * input_dim.batch() * class_num);
-
- vec_arr[0] = vec;
- veclabel_arr[0] = veclabel;
-
- while ((*running)) {
- endflag = false;
- NN_EXCEPTION_NOTI(DATA_NOT_READY);
- if (buf_size - (*cur_size) > 0) {
- /** @todo Update to support multiple inputs later */
- status = callback(vec_arr, veclabel_arr, &endflag, user_data);
- if (endflag) {
- NN_EXCEPTION_NOTI(DATA_END);
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
- return;
- }
- if (status != ML_ERROR_NONE) {
- NN_EXCEPTION_NOTI(DATA_ERROR);
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
- return;
- }
-
- for (unsigned int i = 0; i < input_dim.batch(); ++i) {
- std::vector<float> v;
- std::vector<float> vl;
- unsigned int I =
- i * input_dim.channel() * input_dim.height() * input_dim.width();
- for (unsigned int j = 0; j < input_dim.channel(); ++j) {
- unsigned int J = j * input_dim.height() * input_dim.width();
- for (unsigned int k = 0; k < input_dim.height() * input_dim.width();
- ++k) {
- unsigned int K = I + J + k;
- v.push_back(vec[K]);
- }
- }
-
- I = i * class_num;
- for (unsigned int j = 0; j < class_num; ++j) {
- vl.push_back(veclabel[I + j]);
- }
-
- data_lock.lock();
- data->push_back(v);
- datalabel->push_back(vl);
- (*cur_size)++;
- data_lock.unlock();
- }
- }
- if (buf_size == (*cur_size)) {
- NN_EXCEPTION_NOTI(DATA_READY);
- }
- }
-
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
-}
-
-int DataBufferFromCallback::setProperty(const PropertyType type,
- std::string &value) {
- return DataBuffer::setProperty(type, value);
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file fc_layer.cpp
- * @date 14 May 2020
- * @brief This is Fully Connected Layer Class for Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <fc_layer.h>
-#include <layer_internal.h>
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-enum class FCParams { weight, bias };
-
-int FullyConnectedLayer::initialize() {
- int status = ML_ERROR_NONE;
-
- if (num_inputs != 1) {
- throw std::invalid_argument("Fully connected layer takes only one input");
- }
-
- output_dim[0] = input_dim[0];
- output_dim[0].width(unit);
-
- TensorDim bias_dim = TensorDim();
- bias_dim.setTensorDim(3, unit);
-
- TensorDim dim = output_dim[0];
- dim.height(input_dim[0].width());
- dim.batch(1);
-
- setNumWeights(2);
- weightAt(0) = std::move(Weight(dim, weight_initializer, true, "FC:weight"));
- weightAt(1) = std::move(Weight(bias_dim, bias_initializer, true, "FC:bias"));
-
- return status;
-}
-
-void FullyConnectedLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
- switch (type) {
- case PropertyType::unit: {
- if (!value.empty()) {
- status = setUint(unit, value);
- throw_status(status);
- output_dim[0].width(unit);
- }
- } break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-sharedConstTensors FullyConnectedLayer::forwarding(sharedConstTensors in) {
- Tensor &weight =
- weightAt(static_cast<int>(FCParams::weight)).getVariableRef();
- Tensor &bias = weightAt(static_cast<int>(FCParams::bias)).getVariableRef();
-
- input = *in[0];
- hidden = input.dot(weight);
- hidden.add_i(bias);
-
- if (weight_regularizer == WeightRegularizerType::l2norm) {
- loss = weight_regularizer_constant * 0.5f * (weight.l2norm());
- }
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-void FullyConnectedLayer::read(std::ifstream &file) {
- Layer::read(file);
- if (opt)
- opt->read(file);
-}
-
-void FullyConnectedLayer::save(std::ofstream &file) {
- Layer::save(file);
- if (opt)
- opt->save(file);
-}
-
-void FullyConnectedLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<FullyConnectedLayer> from =
- std::static_pointer_cast<FullyConnectedLayer>(l);
- this->unit = from->unit;
-}
-
-sharedConstTensors
-FullyConnectedLayer::backwarding(sharedConstTensors derivative, int iteration) {
- unsigned int weight_idx = static_cast<int>(FCParams::weight);
- unsigned int bias_idx = static_cast<int>(FCParams::bias);
- Tensor &weight = weightAt(weight_idx).getVariableRef();
- Tensor &djdw = weightAt(weight_idx).getGradientRef();
- Tensor &djdb = weightAt(bias_idx).getGradientRef();
-
- Tensor ret = derivative[0]->dot(weight, false, true);
- djdb = derivative[0]->sum(0);
-
- djdw = input.dot(*derivative[0], true, false);
- if (isWeightRegularizerL2Norm())
- djdw.add_i(weight, weight_regularizer_constant);
- djdw = djdw.sum(0);
-
- if (trainable) {
- opt->apply_gradients(weight_list, num_weights, iteration);
- }
-
- return {MAKE_SHARED_TENSOR(std::move(ret))};
-}
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file flatten_layer.cpp
- * @date 16 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Flatten Layer Class for Neural Network
- *
- */
-
-#include <flatten_layer.h>
-#include <layer_internal.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int FlattenLayer::initialize() {
- if (num_inputs != 1) {
- throw std::invalid_argument("input_shape keyword is only for one input");
- }
-
- TensorDim &out_dim = output_dim[0];
- int status = ML_ERROR_NONE;
- if (input_dim[0].getDataLen() == 1) {
- ml_logw("Warning: the length of previous layer dimension is one");
- }
-
- out_dim.batch(input_dim[0].batch());
- out_dim.channel(1);
- out_dim.height(1);
- out_dim.width(input_dim[0].getFeatureLen());
-
- return status;
-}
-
-sharedConstTensors FlattenLayer::forwarding(sharedConstTensors in) {
- input = *in[0];
- hidden = input;
-
- hidden.reshape(output_dim[0]);
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors FlattenLayer::backwarding(sharedConstTensors in,
- int iteration) {
- Tensor temp = *in[0];
- temp.reshape(input_dim[0]);
-
- return {MAKE_SHARED_TENSOR(std::move(temp))};
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file input_layer.cpp
- * @date 14 May 2020
- * @brief This is Input Layer Class for Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <input_layer.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-
-namespace nntrainer {
-
-void InputLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::normalization:
- if (!value.empty()) {
- status = setBoolean(normalization, value);
- throw_status(status);
- }
- break;
- case PropertyType::standardization:
- if (!value.empty()) {
- status = setBoolean(standardization, value);
- throw_status(status);
- }
- break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-sharedConstTensors InputLayer::forwarding(sharedConstTensors in) {
- input = *in[0];
-
- hidden = input;
- if (normalization)
- hidden = hidden.normalization();
- if (standardization)
- hidden = hidden.standardization();
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors InputLayer::backwarding(sharedConstTensors in,
- int iteration) {
- return in;
-}
-
-int InputLayer::initialize() {
- int status = ML_ERROR_NONE;
- output_dim = input_dim;
-
- return status;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file layer.cpp
- * @date 04 December 2019
- * @brief This is Layers Classes for Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <layer_internal.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <optimizer_factory.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-void Layer::setActivation(ActivationType acti) {
- if (acti == ActivationType::ACT_UNKNOWN) {
- throw std::invalid_argument("Error:have to specify activation function");
- }
- activation_type = acti;
-}
-
-int Layer::setOptimizer(std::shared_ptr<Optimizer> opt) {
- this->opt = createOptimizer(opt->getType(), *opt.get());
- return this->opt->initialize(weight_list, num_weights, true);
-}
-
-int Layer::checkValidation() {
- int status = ML_ERROR_NONE;
- if (type == LayerType::LAYER_UNKNOWN) {
- ml_loge("Error: Layer type is unknown");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- if (activation_type == ActivationType::ACT_UNKNOWN) {
- ml_loge("Error: Have to set activation for this layer");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- return status;
-}
-
-void Layer::setBatch(unsigned int batch) {
- for (unsigned int idx = 0; idx < num_inputs; ++idx)
- input_dim[idx].setTensorDim(0, batch);
-
- for (unsigned int idx = 0; idx < num_outputs; ++idx)
- output_dim[idx].setTensorDim(0, batch);
-}
-
-void Layer::copy(std::shared_ptr<Layer> l) {
- setNumWeights(l->num_weights);
- for (unsigned int i = 0; i < num_weights; ++i) {
- weightAt(i) = l->weightAt(i);
- }
-
- // TODO: fix this #630
- this->opt = l->opt;
- this->input_dim = l->input_dim;
- this->output_dim = l->output_dim;
- this->input.copy(l->input);
- this->hidden.copy(l->hidden);
- this->activation_type = l->activation_type;
- this->loss = l->loss;
- this->type = l->type;
- this->weight_regularizer = l->weight_regularizer;
- this->weight_regularizer_constant = l->weight_regularizer_constant;
- this->weight_initializer = l->weight_initializer;
- this->flatten = l->flatten;
- this->trainable = l->trainable;
- this->num_inputs = l->num_inputs;
- this->num_outputs = l->num_outputs;
-}
-
-void Layer::read(std::ifstream &file) {
- for (unsigned int i = 0; i < num_weights; ++i) {
- weightAt(i).getVariableRef().read(file);
- }
-}
-
-void Layer::save(std::ofstream &file) {
- for (unsigned int i = 0; i < num_weights; ++i) {
- weightAt(i).getVariableRef().save(file);
- }
-}
-
-int Layer::setProperty(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
-
- status = getKeyValue(values[i], key, value);
- NN_RETURN_STATUS();
-
- unsigned int type = parseLayerProperty(key);
-
- if (value.empty()) {
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- try {
- /// @note this calls derived setProperty if available
- setProperty(static_cast<PropertyType>(type), value);
- } catch (...) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
- return status;
-}
-
-void Layer::setProperty(const PropertyType type, const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::name:
- if (!value.empty()) {
- status = setName(value);
- throw_status(status);
- }
- break;
- case PropertyType::input_shape: {
- if (num_inputs != 1) {
- throw std::invalid_argument("input_shape keyword is only for one input");
- }
-
- TensorDim &in_dim = input_dim[0];
- if (!value.empty()) {
- unsigned int cache_batch_size = 1;
- /** cache original value of batch size */
- if (in_dim.batch()) {
- cache_batch_size = in_dim.batch();
- in_dim.batch(1);
- }
- status = in_dim.setTensorDim(value.c_str());
- if (in_dim.batch() > 1) {
- ml_logw("Batch size set with input dimension %d is ignored."
- "Set batchsize property for the model to update batchsize.",
- in_dim.batch());
- }
- /** set back to cache value of dimension */
- in_dim.batch(cache_batch_size);
- throw_status(status);
- }
- } break;
- case PropertyType::activation:
- if (!value.empty()) {
- setActivation((ActivationType)parseType(value, TOKEN_ACTI));
- }
- break;
- case PropertyType::flatten:
- if (!value.empty()) {
- status = setBoolean(flatten, value);
- throw_status(status);
- }
- break;
- case PropertyType::weight_regularizer:
- if (!value.empty()) {
- weight_regularizer =
- (WeightRegularizerType)parseType(value, TOKEN_WEIGHT_REGULARIZER);
- if (weight_regularizer == WeightRegularizerType::unknown) {
- throw std::invalid_argument("[Layer] Unknown Weight decay");
- }
- }
- break;
- case PropertyType::weight_regularizer_constant:
- if (!value.empty()) {
- status = setFloat(weight_regularizer_constant, value);
- throw_status(status);
- }
- break;
- case PropertyType::weight_initializer:
- if (!value.empty()) {
- weight_initializer =
- (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
- }
- break;
- case PropertyType::bias_initializer:
- if (!value.empty()) {
- bias_initializer = (WeightInitializer)parseType(value, TOKEN_WEIGHT_INIT);
- }
- break;
- case PropertyType::input_layers:
- if (!value.empty()) {
- std::regex reg("\\,+");
- std::vector<std::string> concat_layers = split(value, reg);
- // TODO set num_inputs properly
- num_inputs = 1;
- if (concat_layers.size() > 1)
- num_inputs = concat_layers.size();
- }
- break;
- default:
- std::string msg =
- "[Layer] Unknown Layer Property Key for value " + std::string(value);
- throw exception::not_supported(msg);
- }
-}
-
-int Layer::setName(std::string name_) {
- if (name_.empty())
- return ML_ERROR_INVALID_PARAMETER;
-
- name = name_;
- return ML_ERROR_NONE;
-}
-
-template <typename T>
-void Layer::printIfValid(std::ostream &out, const PropertyType type,
- const T target) {
- try {
- setProperty(type);
- } catch (exception::not_supported &e) {
- return;
- }
-
- out << propToStr(static_cast<unsigned int>(type)) << ": " << target
- << std::endl;
-}
-
-void Layer::printShapeInfo(std::ostream &out) {
- for (unsigned int idx = 0; idx < num_inputs; ++idx) {
- out << "input " << input_dim[idx];
- for (unsigned int i = 0; i < num_weights; i++)
- out << "inner" << i << " " << weightAt(i).var.getDim();
- }
- for (unsigned int idx = 0; idx < num_outputs; ++idx) {
- out << "output " << output_dim[idx];
- }
-}
-
-void Layer::printPropertiesMeta(std::ostream &out) {
- printIfValid(
- out, PropertyType::activation,
- static_cast<std::underlying_type<ActivationType>::type>(activation_type));
- printIfValid(out, PropertyType::flatten, flatten);
-}
-
-void Layer::printProperties(std::ostream &out) {
- out << "Trainable: " << trainable << std::endl;
- printIfValid(out, PropertyType::weight_regularizer,
- static_cast<int>(weight_regularizer));
- printIfValid(out, PropertyType::weight_regularizer_constant,
- weight_regularizer_constant);
-}
-
-void Layer::printMetric(std::ostream &out) {
- if (loss > 0) {
- out << "Weight regularization loss: " << loss;
- }
-}
-
-void Layer::printPreset(std::ostream &out, PrintPreset preset) {
- unsigned int flags = 0;
- switch (preset) {
- case PrintPreset::PRINT_ALL:
- flags = PRINT_WEIGHTS | PRINT_METRIC;
- /// fall through intended
- case PrintPreset::PRINT_SUMMARY_META:
- flags |= PRINT_PROP_META;
- /// fall through intended
- case PrintPreset::PRINT_SUMMARY:
- flags |= PRINT_INST_INFO | PRINT_SHAPE_INFO | PRINT_PROP | PRINT_PROP_META;
- break;
- case PrintPreset::PRINT_NONE:
- return;
- default:
- throw ::std::invalid_argument("undefined preset given");
- }
- print(out, flags);
-}
-
-void Layer::print(std::ostream &out, unsigned int flags) {
- if (flags & PRINT_INST_INFO) {
- out << "===================";
- if (getName().empty())
- printInstance(out, this);
- else
- out << "<" << getName() << ">" << std::endl;
-
- out << "Layer Type: "
- << static_cast<std::underlying_type<LayerType>::type>(type)
- << std::endl;
- }
-
- if (flags & PRINT_SHAPE_INFO) {
- out << "======shape information: " << std::endl;
- printShapeInfo(out);
- }
-
- if (flags & PRINT_PROP_META) {
- out << "======meta properties: " << std::endl;
- printPropertiesMeta(out);
- }
-
- if (flags & PRINT_PROP) {
- out << "======properties: " << std::endl;
- printProperties(out);
- }
-
- if (flags & PRINT_WEIGHTS) {
- out << "======weights: " << std::endl;
- for (unsigned int i = 0; i < num_weights; ++i) {
- out << '[' << weightAt(i).getName() << ']' << std::endl;
- out << weightAt(i).var;
- }
- }
-
- if (flags & PRINT_METRIC) {
- out << "======metrics: " << std::endl;
- printMetric(out);
- }
-};
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file optimizer_factory.cpp
- * @date 11 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the layer factory.
- */
-
-#include <layer_factory.h>
-
-#include <activation_layer.h>
-#include <addition_layer.h>
-#include <bn_layer.h>
-#include <conv2d_layer.h>
-#include <fc_layer.h>
-#include <flatten_layer.h>
-#include <input_layer.h>
-#include <loss_layer.h>
-#include <nntrainer_error.h>
-#include <pooling2d_layer.h>
-
-#ifdef ENABLE_TFLITE_BACKBONE
-#include <tflite_layer.h>
-#endif
-
-#ifdef ENABLE_NNSTREAMER_BACKBONE
-#include <nnstreamer_layer.h>
-#endif
-
-namespace nntrainer {
-
-/**
- * @brief Factory creator with constructor
- */
-std::unique_ptr<Layer> createLayer(LayerType type) {
- switch (type) {
- case LayerType::LAYER_IN:
- return std::make_unique<InputLayer>();
- case LayerType::LAYER_FC:
- return std::make_unique<FullyConnectedLayer>();
- case LayerType::LAYER_BN:
- return std::make_unique<BatchNormalizationLayer>();
- case LayerType::LAYER_CONV2D:
- return std::make_unique<Conv2DLayer>();
- case LayerType::LAYER_POOLING2D:
- return std::make_unique<Pooling2DLayer>();
- case LayerType::LAYER_FLATTEN:
- return std::make_unique<FlattenLayer>();
- case LayerType::LAYER_ACTIVATION:
- return std::make_unique<ActivationLayer>();
- case LayerType::LAYER_ADDITION:
- return std::make_unique<AdditionLayer>();
- case LayerType::LAYER_LOSS:
- return std::make_unique<LossLayer>();
- case LayerType::LAYER_BACKBONE_NNSTREAMER:
-#ifdef ENABLE_NNSTREAMER_BACKBONE
- return std::make_unique<NNStreamerLayer>();
-#else
- throw exception::not_supported("NNStreamer backbone layer not supported");
-#endif
- case LayerType::LAYER_BACKBONE_TFLITE:
-#ifdef ENABLE_TFLITE_BACKBONE
- return std::make_unique<TfLiteLayer>();
-#else
- throw exception::not_supported("TfLite backbone layer not supported");
-#endif
- case LayerType::LAYER_UNKNOWN:
- /** fallthrough intended */
- default:
- throw std::invalid_argument("Unknown type for the layer");
- }
-}
-
-} // namespace nntrainer
+++ /dev/null
-/* SPDX-License-Identifier: Apache-2.0
- *
- * Copyright (C) 2020 Jihoon Lee <jihoon.it.lee@samsung.com>
- *
- * @file lazy_tensor.cpp
- * @date 05 Jun 2020
- * @brief A lazy evaluation calculator for tensors
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jihoon Lee <jihoon.it.lee@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-
-namespace nntrainer {
-
-/**
- * @brief Wrapper method of add_i (immediate version of add)
- * @retval this
- */
-LazyTensor &LazyTensor::add_i(float const &value) {
- call_chain.push_back(
- [value](Tensor &t) mutable -> int { return t.add_i(value); });
- return *this;
-}
-/**
- * @brief Wrapper method of add_i. see tensor.h for more detail
- * @param[in] m Tensor to be added
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::add_i(Tensor const &m, float const alpha) {
- auto f = [&m, alpha](Tensor &t) mutable -> int { return t.add_i(m, alpha); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of subtract_i. see tensor.h for more detail
- * @param[in] m Tensor to subtract
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::subtract_i(Tensor const &m) {
- auto f = [&m](Tensor &t) mutable -> int { return t.subtract_i(m); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of subtract_i. see tensor.h for more detail
- * @param[in] value value to subtract
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::subtract_i(float const &value) {
- auto f = [value](Tensor &t) mutable -> int { return t.subtract_i(value); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of multiply_i. see tensor.h for more detail
- * @param[in] value to be added
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::multiply_i(float const &value) {
- auto f = [value](Tensor &t) mutable -> int { return t.multiply_i(value); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of multiply_i. see tensor.h for more detail
- * @param[in] m Tensor to be multiplied
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::multiply_i(Tensor const &m) {
- auto f = [&m](Tensor &t) mutable -> int { return t.multiply_i(m); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of divide_i. see tensor.h for more detail
- * @param[in] value divisor
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::divide_i(float const &value) {
- auto f = [value](Tensor &t) mutable -> int { return t.divide_i(value); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of divide_i. see tensor.h for more detail
- * @param[in] m Tensor to for division
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::divide_i(Tensor const &m) {
- auto f = [&m](Tensor &t) mutable -> int { return t.divide_i(m); };
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of dot. see tensor.h for more detail (memcopy
- * happens)
- * @param[in] m Tensor
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::dot(Tensor const &m) {
- auto f = [&m](Tensor &t) mutable -> int {
- try {
- t = t.dot(m);
- return ML_ERROR_NONE;
- } catch (std::runtime_error &e) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- };
-
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of transpose. see tensor.h for more detail (memcopy
- * happens)
- * @param[in] direction to transpose ex) 0:2:1
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::transpose(std::string direction) {
- auto f = [direction](Tensor &t) mutable -> int {
- try {
- t = t.transpose(direction);
- return ML_ERROR_NONE;
- } catch (std::runtime_error &e) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- };
-
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of sum. see tensor.h for more detail (memcopy
- * happens)
- * @param[in] direction to transpose ex) 0:2:1
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::sum_by_batch() {
- auto f = [](Tensor &t) mutable -> int {
- try {
- t = t.sum_by_batch();
- return ML_ERROR_NONE;
- } catch (std::runtime_error &e) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- };
-
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of sum. see tensor.h for more detail (memcopy
- * happens) 0 : batch direction 1 : channel direction 2 : channel direction 3 :
- * channel direction
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::sum(int axis) {
- auto f = [axis](Tensor &t) mutable -> int {
- try {
- t = t.sum(axis);
- return ML_ERROR_NONE;
- } catch (std::runtime_error &e) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- };
-
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of average. see tensor.h for more detail (memcopy
- * happens)
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::average(int axis) {
- auto f = [axis](Tensor &t) mutable -> int {
- try {
- t = t.average(axis);
- return ML_ERROR_NONE;
- } catch (std::runtime_error &e) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- };
-
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief Wrapper method of average. see tensor.h for more detail (memcopy
- * happens)
- * @retval LazyTensor *this
- */
-LazyTensor &LazyTensor::average() {
- auto f = [](Tensor &t) mutable -> int {
- try {
- t = t.average();
- return ML_ERROR_NONE;
- } catch (std::runtime_error &e) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- };
-
- call_chain.push_back(f);
- return *this;
-}
-
-/**
- * @brief execute the call_chain to evaluate
- * @retval calculated tensor
- */
-Tensor LazyTensor::run() {
- int status;
- for (auto item : call_chain) {
- status = item(target);
- if (status != ML_ERROR_NONE) {
- throw std::runtime_error("Error: evaluation failed");
- }
- }
- return target;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file loss_layer.cpp
- * @date 12 June 2020
- * @brief This is Loss Layer Class for Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <activation_layer.h>
-#include <cmath>
-#include <layer_internal.h>
-#include <lazy_tensor.h>
-#include <loss_layer.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int LossLayer::initialize() {
- int status = ML_ERROR_NONE;
-
- output_dim = input_dim;
- return status;
-}
-
-sharedConstTensors LossLayer::forwarding(sharedConstTensors in,
- sharedConstTensors label) {
- input = *in[0];
- Tensor y2 = *label[0];
- Tensor y = input;
- Tensor l;
-
- switch (loss_type) {
- case LossType::LOSS_MSE: {
- // y2 <- y2 - y;
- Tensor residual = y2.subtract(y);
- l = residual.chain().multiply_i(residual).average().run();
- } break;
- case LossType::LOSS_ENTROPY_SIGMOID: {
- // @todo: change this to apply_i
- // @note: the output should be logit before applying sigmoid
- // log(1 + exp(-abs(y))) + max(y, 0)
- Tensor mid_term = y.apply(static_cast<float (*)(float)>(&std::fabs))
- .multiply(-1.0)
- .apply(static_cast<float (*)(float)>(&std::exp))
- .add(1.0)
- .apply(logFloat);
- mid_term = mid_term.add(y.apply(ActivationLayer::relu));
-
- // y * y2
- Tensor end_term = y2.chain().multiply_i(y).run();
-
- // loss = log(1 + exp(-abs(y))) + max(y, 0) - (y * y2)
- l = mid_term.subtract(end_term).average();
- y = y.apply(ActivationLayer::sigmoid);
- } break;
- case LossType::LOSS_ENTROPY_SOFTMAX: {
- y = y.apply(ActivationLayer::softmax);
- l = y2.chain()
- .multiply_i(y.apply(logFloat))
- .run()
- .sum_by_batch()
- .multiply(-1);
-
- } break;
- case LossType::LOSS_ENTROPY: {
- throw std::runtime_error(
- "Error: Cross Entropy not supported without softmax or sigmoid.");
- }
- case LossType::LOSS_UNKNOWN:
- /** intended */
- default: { throw std::runtime_error("Error: Unknown loss_type."); }
- }
-
- updateLoss(l);
- return {MAKE_SHARED_TENSOR(std::move(y))};
-}
-
-sharedConstTensors LossLayer::forwarding(sharedConstTensors in) {
- Tensor ret;
-
- switch (loss_type) {
- case LossType::LOSS_MSE:
- return in;
- case LossType::LOSS_ENTROPY_SIGMOID:
- ret = in[0]->apply(ActivationLayer::sigmoid);
- return {MAKE_SHARED_TENSOR(std::move(ret))};
- case LossType::LOSS_ENTROPY_SOFTMAX:
- ret = in[0]->apply(ActivationLayer::softmax);
- return {MAKE_SHARED_TENSOR(std::move(ret))};
- case LossType::LOSS_ENTROPY:
- throw std::runtime_error(
- "Error: Cross Entropy not supported without softmax or sigmoid.");
- case LossType::LOSS_UNKNOWN:
- /** intended */
- default:
- throw std::runtime_error("Error: Unknown loss_type.");
- }
-}
-
-void LossLayer::updateLoss(const Tensor &l) {
- float loss_sum = 0.0f;
- const float *data = l.getData();
-
- for (unsigned int i = 0; i < l.batch(); i++) {
- loss_sum += data[i];
- }
- loss = loss_sum / (float)l.batch();
-}
-
-void LossLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<LossLayer> from = std::static_pointer_cast<LossLayer>(l);
- this->loss_type = from->loss_type;
-}
-
-sharedConstTensors LossLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- Tensor ret_derivative;
- Tensor y2 = *derivative[0];
- Tensor y = input;
-
- switch (loss_type) {
- case LossType::LOSS_MSE:
- ret_derivative = y.subtract(y2).multiply(2).divide(y.getDim().getDataLen());
- break;
- case LossType::LOSS_ENTROPY_SIGMOID:
- y = y.apply(ActivationLayer::sigmoid);
- ret_derivative = y.subtract(y2).divide(y.getDim().getDataLen());
- break;
- case LossType::LOSS_ENTROPY_SOFTMAX:
- y = y.apply(ActivationLayer::softmax);
- ret_derivative = y.subtract(y2).divide(y.batch());
- break;
- case LossType::LOSS_ENTROPY:
- throw std::runtime_error(
- "Error: Cross Entropy not supported without softmax or sigmoid.");
- case LossType::LOSS_UNKNOWN:
- /** intended */
- default:
- throw std::runtime_error("Unknown loss_type.");
- }
-
- return {MAKE_SHARED_TENSOR(std::move(ret_derivative))};
-}
-
-int LossLayer::setLoss(LossType l) {
- int status = ML_ERROR_NONE;
- if (l == LossType::LOSS_UNKNOWN) {
- ml_loge("Error: Unknown loss type");
- return ML_ERROR_INVALID_PARAMETER;
- }
- loss_type = l;
- return status;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file model_loader.c
- * @date 5 August 2020
- * @brief This is model loader class for the Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <adam.h>
-#include <databuffer_factory.h>
-#include <databuffer_file.h>
-#include <databuffer_func.h>
-#include <layer_factory.h>
-#include <model_loader.h>
-#include <neuralnet.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <optimizer_factory.h>
-#include <parse_util.h>
-#include <sstream>
-#include <util_func.h>
-
-#define NN_INI_RETURN_STATUS() \
- do { \
- if (status != ML_ERROR_NONE) { \
- iniparser_freedict(ini); \
- return status; \
- } \
- } while (0)
-
-namespace nntrainer {
-
-/**
- * @brief load model config from ini
- */
-int ModelLoader::loadModelConfigIni(dictionary *ini, NeuralNetwork &model) {
- int status = ML_ERROR_NONE;
-
- if (iniparser_find_entry(ini, "Model") == 0) {
- ml_loge("there is no [Model] section in given ini file");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- /** Default to neural network model type */
- model.net_type = (nntrainer::NetType)parseType(
- iniparser_getstring(ini, "Model:Type", unknown), TOKEN_MODEL);
- model.epochs = iniparser_getint(ini, "Model:Epochs", model.epochs);
- model.loss_type = (LossType)parseType(
- iniparser_getstring(ini, "Model:Loss", unknown), TOKEN_LOSS);
- model.save_path = iniparser_getstring(ini, "Model:Save_path", "./model.bin");
- model.batch_size =
- iniparser_getint(ini, "Model:Batch_Size", model.batch_size);
-
- /** Default to adam optimizer */
- OptType opt_type = (OptType)parseType(
- iniparser_getstring(ini, "Model:Optimizer", "adam"), TOKEN_OPT);
-
- try {
- model.opt = nntrainer::createOptimizer(opt_type);
- } catch (std::exception &e) {
- ml_loge("%s %s", typeid(e).name(), e.what());
- return ML_ERROR_INVALID_PARAMETER;
- } catch (...) {
- ml_loge("Creating the optimizer failed");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- std::vector<std::string> optimizer_prop = {};
- optimizer_prop.push_back(
- {"learning_rate=" +
- std::string(iniparser_getstring(
- ini, "Model:Learning_rate",
- std::to_string(model.opt->getLearningRate()).c_str()))});
-
- optimizer_prop.push_back(
- {"decay_steps=" + std::string(iniparser_getstring(
- ini, "Model:Decay_steps",
- std::to_string(model.opt->getDecaySteps()).c_str()))});
- optimizer_prop.push_back(
- {"decay_rate=" + std::string(iniparser_getstring(
- ini, "Model:Decay_rate",
- std::to_string(model.opt->getDecayRate()).c_str()))});
-
- if (model.opt->getType() == OptType::ADAM) {
- std::shared_ptr<Adam> opt_adam = std::static_pointer_cast<Adam>(model.opt);
-
- optimizer_prop.push_back(
- {"beta1=" +
- std::string(iniparser_getstring(
- ini, "Model:Beta1", std::to_string(opt_adam->getBeta1()).c_str()))});
- optimizer_prop.push_back(
- {"beta2=" +
- std::string(iniparser_getstring(
- ini, "Model:Beta2", std::to_string(opt_adam->getBeta2()).c_str()))});
- optimizer_prop.push_back(
- {"epsilon=" + std::string(iniparser_getstring(
- ini, "Model:Epsilon",
- std::to_string(opt_adam->getEpsilon()).c_str()))});
- }
-
- status = model.opt->setProperty(optimizer_prop);
- NN_RETURN_STATUS();
-
- return status;
-}
-
-/**
- * @brief load dataset config from ini
- */
-int ModelLoader::loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model) {
- ml_logd("start parsing dataset config");
- int status = ML_ERROR_NONE;
-
- if (iniparser_find_entry(ini, "Dataset") == 0) {
- model.data_buffer = nntrainer::createDataBuffer(DataBufferType::GENERATOR);
- status = model.data_buffer->setBatchSize(model.batch_size);
- return status;
- }
-
- if (iniparser_find_entry(ini, "DataSet:Tflite")) {
- ml_loge("Error: Tflite dataset is not yet implemented!");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- model.data_buffer = nntrainer::createDataBuffer(DataBufferType::FILE);
- std::shared_ptr<DataBufferFromDataFile> dbuffer =
- std::static_pointer_cast<DataBufferFromDataFile>(model.data_buffer);
-
- std::function<int(const char *, DataType, bool)> parse_and_set =
- [&](const char *key, DataType dt, bool required) -> int {
- const char *path = iniparser_getstring(ini, key, NULL);
-
- if (path == NULL) {
- return required ? ML_ERROR_INVALID_PARAMETER : ML_ERROR_NONE;
- }
-
- return dbuffer->setDataFile(dt, path);
- };
-
- status = parse_and_set("DataSet:TrainData", DATA_TRAIN, true);
- NN_RETURN_STATUS();
- status = parse_and_set("DataSet:ValidData", DATA_VAL, false);
- NN_RETURN_STATUS();
- status = parse_and_set("DataSet:TestData", DATA_TEST, false);
- NN_RETURN_STATUS();
- status = parse_and_set("Dataset:LabelData", DATA_LABEL, true);
- NN_RETURN_STATUS();
-
- status = model.data_buffer->setBatchSize(model.batch_size);
- NN_RETURN_STATUS();
-
- unsigned int bufsize = iniparser_getint(ini, "DataSet:BufferSize", 1);
- status = model.data_buffer->setBufSize(bufsize);
- NN_RETURN_STATUS();
-
- ml_logd("parsing dataset done");
- return status;
-}
-
-int ModelLoader::loadLayerConfigIniCommon(dictionary *ini,
- std::shared_ptr<Layer> &layer,
- const std::string &layer_name,
- LayerType layer_type) {
- int status = ML_ERROR_NONE;
-
- try {
- layer = nntrainer::createLayer(layer_type);
- } catch (const std::exception &e) {
- ml_loge("%s %s", typeid(e).name(), e.what());
- status = ML_ERROR_INVALID_PARAMETER;
- } catch (...) {
- ml_loge("unknown error type thrown");
- status = ML_ERROR_INVALID_PARAMETER;
- }
- NN_RETURN_STATUS();
-
- unsigned int property_end =
- static_cast<unsigned int>(Layer::PropertyType::unknown);
-
- for (unsigned int i = 0; i < property_end; ++i) {
- std::string prop = propToStr(i);
- std::string value =
- iniparser_getstring(ini, (layer_name + ":" + prop).c_str(), unknown);
-
- /**! @todo: add following negative tc after #319
- * 1. layer has empty prop -> throw std::invalid_argument
- * 2. layer has not allowed property -> throw exception::not_supported
- * 3. property value parse error -> throw std::invalid_argument
- */
- if (!strncmp(value.c_str(), unknown, strlen(unknown))) {
- continue;
- }
-
- if (value == "") {
- std::stringstream ss;
- ss << "property key " << prop << " has empty value. It is not allowed";
- throw std::invalid_argument(ss.str());
- }
-
- layer->setProperty(static_cast<Layer::PropertyType>(i), value);
- }
-
- status = layer->setName(layer_name);
- NN_RETURN_STATUS();
-
- return ML_ERROR_NONE;
-}
-
-int ModelLoader::loadLayerConfigIni(dictionary *ini,
- std::shared_ptr<Layer> &layer,
- const std::string &layer_name) {
- std::string layer_type_str =
- iniparser_getstring(ini, (layer_name + ":Type").c_str(), unknown);
- LayerType layer_type = (LayerType)parseType(layer_type_str, TOKEN_LAYER);
-
- return loadLayerConfigIniCommon(ini, layer, layer_name, layer_type);
-}
-
-int ModelLoader::loadBackboneConfigIni(dictionary *ini,
- const std::string &backbone_config,
- NeuralNetwork &model,
- const std::string &backbone_name) {
- int status = ML_ERROR_NONE;
- NeuralNetwork backbone;
-
- bool trainable =
- iniparser_getboolean(ini, (backbone_name + ":trainable").c_str(), false);
-
- status = loadFromConfig(backbone_config, backbone, true);
- NN_RETURN_STATUS();
-
- auto graph = backbone.getGraph();
- for (auto &layer : graph)
- layer->setTrainable(trainable);
-
- status = model.extendGraph(backbone.getGraph(), backbone_name);
- NN_RETURN_STATUS();
-
- return ML_ERROR_NONE;
-}
-
-int ModelLoader::loadBackboneConfigExternal(dictionary *ini,
- const std::string &backbone_config,
- std::shared_ptr<Layer> &layer,
- const std::string &backbone_name) {
- LayerType type = LayerType::LAYER_UNKNOWN;
-
-#if defined(ENABLE_NNSTREAMER_BACKBONE)
- type = LayerType::LAYER_BACKBONE_NNSTREAMER;
-#endif
-
- /** TfLite has higher priority */
-#if defined(ENABLE_TFLITE_BACKBONE)
- if (fileTfLite(backbone_config))
- type = LayerType::LAYER_BACKBONE_TFLITE;
-#endif
-
- if (type == LayerType::LAYER_UNKNOWN)
- return ML_ERROR_NOT_SUPPORTED;
-
- int status = ML_ERROR_NONE;
- status = loadLayerConfigIniCommon(ini, layer, backbone_name, type);
- NN_RETURN_STATUS();
-
- layer->setProperty(Layer::PropertyType::modelfile, backbone_config);
- return status;
-}
-
-/**
- * @brief load all of model and dataset from ini
- */
-int ModelLoader::loadFromIni(std::string ini_file, NeuralNetwork &model,
- bool bare_layers) {
- int status = ML_ERROR_NONE;
- int num_ini_sec = 0;
- dictionary *ini;
- const char model_str[] = "model";
- unsigned int model_len = strlen(model_str);
- const char dataset_str[] = "dataset";
- unsigned int dataset_len = strlen(dataset_str);
-
- if (ini_file.empty()) {
- ml_loge("Error: Configuration File is not defined");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- if (!isFileExist(ini_file)) {
- ml_loge("Cannot open model configuration file, filename : %s",
- ini_file.c_str());
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- /** Parse ini file */
- ini = iniparser_load(ini_file.c_str());
- if (ini == NULL) {
- ml_loge("Error: cannot parse file: %s\n", ini_file.c_str());
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- /** Get number of sections in the file */
- num_ini_sec = iniparser_getnsec(ini);
- if (num_ini_sec < 0) {
- ml_loge("Error: invalid number of sections.");
- status = ML_ERROR_INVALID_PARAMETER;
- NN_INI_RETURN_STATUS();
- }
-
- if (!bare_layers) {
- status = loadModelConfigIni(ini, model);
- NN_INI_RETURN_STATUS();
-
- status = loadDatasetConfigIni(ini, model);
- NN_INI_RETURN_STATUS();
- }
-
- ml_logd("parsing ini started");
- /** Get all the section names */
- ml_logi("==========================parsing ini...");
- ml_logi("invalid properties does not cause error, rather be ignored");
- ml_logi("not-allowed property for the layer throws error");
- ml_logi("valid property with invalid value throws error as well");
- for (int idx = 0; idx < num_ini_sec; ++idx) {
- std::string sec_name = iniparser_getsecname(ini, idx);
- ml_logd("probing section name: %s", sec_name.c_str());
-
- if (sec_name.empty()) {
- ml_loge("Error: Unable to retrieve section names from ini.");
- status = ML_ERROR_INVALID_PARAMETER;
- NN_INI_RETURN_STATUS();
- }
-
- if (strncasecmp(model_str, sec_name.c_str(), model_len) == 0)
- continue;
-
- if (strncasecmp(dataset_str, sec_name.c_str(), dataset_len) == 0)
- continue;
-
- /** Parse all the layers defined as sections in order */
- std::shared_ptr<Layer> layer;
-
- /**
- * If this section is a backbone, load backbone section from this
- * @note The order of backbones in the ini file defines the order on the
- * backbones in the model graph
- */
- const char *backbone =
- iniparser_getstring(ini, (sec_name + ":Backbone").c_str(), unknown);
- if (backbone == unknown) {
- status = loadLayerConfigIni(ini, layer, sec_name);
- } else if (fileIni(backbone)) {
- status = loadBackboneConfigIni(ini, backbone, model, sec_name);
- NN_INI_RETURN_STATUS();
- continue;
- } else {
- status = loadBackboneConfigExternal(ini, backbone, layer, sec_name);
- }
- NN_INI_RETURN_STATUS();
-
- status = model.addLayer(layer);
- NN_INI_RETURN_STATUS();
- }
- ml_logd("parsing ini finished");
-
- if (model.layers.empty()) {
- ml_loge("there is no layer section in the ini file");
- status = ML_ERROR_INVALID_PARAMETER;
- }
-
- iniparser_freedict(ini);
- return status;
-}
-
-/**
- * @brief load all of model and dataset from given config file
- */
-int ModelLoader::loadFromConfig(std::string config, NeuralNetwork &model) {
- return loadFromConfig(config, model, false);
-}
-
-/**
- * @brief load all of model and dataset from given config file
- */
-int ModelLoader::loadFromConfig(std::string config, NeuralNetwork &model,
- bool bare_layers) {
- if (fileIni(config)) {
- return loadFromIni(config, model, bare_layers);
- }
-
- return ML_ERROR_INVALID_PARAMETER;
-}
-
-bool ModelLoader::fileExt(const std::string &filename, const std::string &ext) {
- size_t position = filename.find_last_of(".");
- if (position == std::string::npos)
- return false;
-
- if (filename.substr(position + 1) == ext) {
- return true;
- }
-
- return false;
-}
-
-bool ModelLoader::fileIni(const std::string &filename) {
- return fileExt(filename, "ini");
-}
-
-bool ModelLoader::fileTfLite(const std::string &filename) {
- return fileExt(filename, "tflite");
-}
-
-} // namespace nntrainer
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file neuralnet.cpp
- * @date 04 December 2019
- * @brief This is Neural Network Class
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <cmath>
-#include <fstream>
-#include <sstream>
-
-#include <databuffer_file.h>
-#include <databuffer_func.h>
-#include <model_loader.h>
-#include <neuralnet.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <unordered_set>
-#include <util_func.h>
-
-/**
- * @brief Internal enum values for nntrainer to summarize model accuracy & loss
- */
-#define ML_TRAIN_SUMMARY_MODEL_TRAIN_LOSS 101
-#define ML_TRAIN_SUMMARY_MODEL_VALID_LOSS 102
-#define ML_TRAIN_SUMMARY_MODEL_VALID_ACCURACY 103
-
-namespace nntrainer {
-
-int NeuralNetwork::loadFromConfig(std::string config) {
- if (loadedFromConfig == true) {
- ml_loge("cannnot do loadFromConfig twice");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- ModelLoader loader;
- NeuralNetwork tempNet(*this);
- int status = loader.loadFromConfig(config, tempNet);
- if (status != ML_ERROR_NONE) {
- return status;
- }
-
- tempNet.loadedFromConfig = true;
- swap(tempNet, *this);
-
- return ML_ERROR_NONE;
-}
-
-int NeuralNetwork::initLossLayer() {
- int status = ML_ERROR_NONE;
- LossType updated_loss_type = loss_type;
-
- if (layers.empty()) {
- status = ML_ERROR_INVALID_PARAMETER;
- NN_RETURN_STATUS();
- }
-
- if (updated_loss_type == LossType::LOSS_ENTROPY) {
- if (layers.back()->getType() != LayerType::LAYER_ACTIVATION) {
- ml_loge("Error: Cross Entropy need last layer to have softmax or sigmoid "
- "activation.");
- return ML_ERROR_NOT_SUPPORTED;
- }
-
- NodeType act_layer = layers.back();
- layers.pop_back();
-
- switch (act_layer->getActivationType()) {
- case ActivationType::ACT_SIGMOID:
- updated_loss_type = LossType::LOSS_ENTROPY_SIGMOID;
- break;
- case ActivationType::ACT_SOFTMAX:
- updated_loss_type = LossType::LOSS_ENTROPY_SOFTMAX;
- break;
- default:
- ml_loge("Error: Cross Entropy not supported without softmax or sigmoid.");
- return ML_ERROR_NOT_SUPPORTED;
- }
- }
-
- std::shared_ptr<LossLayer> loss_layer = std::make_shared<LossLayer>();
- ensureName(loss_layer);
-
- loss_layer->setInputDimension(getOutputDimension());
- status = loss_layer->initialize();
- NN_RETURN_STATUS();
-
- status = loss_layer->setLoss(updated_loss_type);
- NN_RETURN_STATUS();
-
- addLayer(std::static_pointer_cast<Layer>(loss_layer));
- return status;
-}
-
-int NeuralNetwork::setProperty(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
- status = getKeyValue(values[i], key, value);
- NN_RETURN_STATUS();
-
- unsigned int type = parseNetProperty(key);
-
- switch (static_cast<PropertyType>(type)) {
- case PropertyType::loss: {
- status = setFloat(loss, value);
- NN_RETURN_STATUS();
- } break;
- case PropertyType::loss_type: {
- loss_type = (LossType)parseType(value, TOKEN_LOSS);
- } break;
- default:
- status = setTrainConfig({values[i]});
- NN_RETURN_STATUS();
- break;
- }
- }
-
- return status;
-}
-
-int NeuralNetwork::setTrainConfig(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
- status = getKeyValue(values[i], key, value);
- NN_RETURN_STATUS();
-
- unsigned int type = parseNetProperty(key);
-
- switch (static_cast<PropertyType>(type)) {
- case PropertyType::epochs: {
- status = setUint(epochs, value);
- NN_RETURN_STATUS();
- } break;
- case PropertyType::save_path: {
- save_path = value;
- } break;
- case PropertyType::continue_train: {
- bool cont_train;
- status = setBoolean(cont_train, value);
- NN_RETURN_STATUS();
- continue_train = cont_train;
- opt->setProperty({values[i]});
- } break;
- case PropertyType::batch_size: {
- status = setUint(batch_size, value);
- NN_RETURN_STATUS();
- /** TODO: increase buffer size if it is smaller than batch size.
- * also if this is set with default batch size, then make it
- * smaller/larger
- */
- } break;
- default:
- ml_loge("Error: Unknown Network Property Key");
- status = ML_ERROR_INVALID_PARAMETER;
- return status;
- }
- }
-
- return status;
-}
-
-int NeuralNetwork::init() {
- int status = ML_ERROR_NONE;
- std::vector<TensorDim> previous_dim;
-
- status = isInitializable();
- NN_RETURN_STATUS();
-
- ml_logd("initiating neural network, layer size: %d",
- (unsigned int)layers.size());
- /** Note: number of entries in layers will change. */
- for (unsigned int i = 0; i < layers.size(); ++i) {
- bool first = i == 0;
- Layer &l = *layers[i];
- ml_logd("layer name: %s", l.getName().c_str());
-
- if (!first) {
- if (layers[i - 1]->getType() == LayerType::LAYER_ACTIVATION &&
- l.getType() == LayerType::LAYER_ACTIVATION) {
- ml_loge("double activation is not allowed");
- return ML_ERROR_INVALID_PARAMETER;
- }
- if (l.getInputDimension().size()) {
- l.setInputDimension(previous_dim);
- } else if (previous_dim != l.getInputDimension()) {
- ml_loge("Dimension mismatch between layers.");
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
-
- status = layers[i]->initialize();
- NN_RETURN_STATUS();
-
- switch (l.getType()) {
- case LayerType::LAYER_BN:
- /// fallthrough intended
- case LayerType::LAYER_CONV2D:
- /// fallthrough intended
- case LayerType::LAYER_FC:
- status = l.setOptimizer(opt);
- NN_RETURN_STATUS();
- break;
- default:
- break;
- }
-
- if (l.getType() != LayerType::LAYER_ACTIVATION) {
- status = realizeActivationType(l.getActivationType(), i);
- NN_RETURN_STATUS();
- }
-
- if (l.getFlatten()) {
- status = realizeFlattenType(i);
- NN_RETURN_STATUS();
- }
-
- previous_dim = l.getOutputDimension();
- }
-
- /** Add the last layer as loss layer */
- status = initLossLayer();
- NN_RETURN_STATUS();
-
- ml_logd("initialize successful, with layer size: %d", (int)layers.size());
-
- for (auto l : layers)
- ml_logd("layer name: %s", l->getName().c_str());
-
- initialized = true;
- setBatchSize(batch_size);
- return status;
-}
-
-/**
- * @brief free layers
- */
-NeuralNetwork::~NeuralNetwork() {
- layers.erase(layers.begin(), layers.end());
-
- if (data_buffer) {
- data_buffer->clear();
- }
-}
-
-/**
- * @brief forward propagation using layers object which has layer
- */
-sharedConstTensors NeuralNetwork::forwarding(sharedConstTensors input) {
- sharedConstTensors X = input;
- /** Do not forward the loss layer, as label is not available */
- for (unsigned int i = 0; i < layers.size() - 1; i++) {
- X = layers[i]->forwarding(X);
- }
-
- return X;
-}
-
-/**
- * @brief forward propagation using layers object which has layer
- */
-sharedConstTensors NeuralNetwork::forwarding(sharedConstTensors input,
- sharedConstTensors label) {
- sharedConstTensors X;
-
- if (input[0]->getDim().batch() > batch_size)
- throw std::logic_error("Error: mismatch in batchsize for data and model.");
-
- X = forwarding(input);
- X = std::static_pointer_cast<LossLayer>(layers[layers.size() - 1])
- ->forwarding(X, label);
-
- return X;
-}
-
-/**
- * @brief back propagation
- * Call backwarding function of layer in reverse order
- * No need to call at first Input Layer (No data to be updated)
- */
-void NeuralNetwork::backwarding(sharedConstTensors input,
- sharedConstTensors label, int iteration) {
-
- if (layers.empty() || layers.back()->getType() != LayerType::LAYER_LOSS) {
- throw std::invalid_argument("last layer is not loss layer");
- }
-
- forwarding(input, label);
-
- sharedConstTensors output = label;
- for (unsigned int i = layers.size() - 1; i > 0; i--)
- output = layers[i]->backwarding(output, iteration);
-}
-
-float NeuralNetwork::getLoss() {
- loss = 0.0f;
- for (unsigned int i = 0; i < layers.size(); i++) {
- loss += layers[i]->getLoss();
- }
- return loss;
-}
-
-void NeuralNetwork::setLoss(float l) { loss = l; }
-
-NeuralNetwork &NeuralNetwork::copy(NeuralNetwork &from) {
- if (this != &from) {
- batch_size = from.batch_size;
- loss = from.loss;
- opt = from.opt;
-
- for (unsigned int i = 0; i < layers.size(); i++)
- layers[i]->copy(from.layers[i]);
- }
- return *this;
-}
-
-/**
- * @brief save model to file
- * save Weight & Bias Data into file by calling save from layer
- * save training parameters from the optimizer
- */
-void NeuralNetwork::saveModel() {
- std::ofstream model_file(save_path, std::ios::out | std::ios::binary);
- for (unsigned int i = 0; i < layers.size(); i++)
- layers[i]->save(model_file);
- model_file.write((char *)&epoch_idx, sizeof(epoch_idx));
- model_file.write((char *)&iter, sizeof(iter));
- model_file.close();
-}
-
-/**
- * @brief read model from file
- * read Weight & Bias Data into file by calling save from layer
- * read training parameters from the optimizer if continuing train
- */
-void NeuralNetwork::readModel() {
- if (!isFileExist(save_path))
- return;
- std::ifstream model_file(save_path, std::ios::in | std::ios::binary);
- for (unsigned int i = 0; i < layers.size(); i++)
- layers[i]->read(model_file);
- if (continue_train) {
- model_file.read((char *)&epoch_idx, sizeof(epoch_idx));
- model_file.read((char *)&iter, sizeof(iter));
- }
- model_file.close();
- ml_logi("read modelfile: %s", save_path.c_str());
-}
-
-void NeuralNetwork::setBatchSize(unsigned int batch) {
- batch_size = batch;
- for (auto const &layer : layers)
- layer->setBatch(batch_size);
-
- if (data_buffer && data_buffer->setBatchSize(batch_size) != ML_ERROR_NONE)
- throw std::invalid_argument("Error setting batchsize for the dataset");
-}
-
-sharedConstTensors NeuralNetwork::inference(sharedConstTensors X) {
- if (batch_size != X[0]->batch()) {
- /**
- * Note that inference resets batch_size of the previous train configuration
- * Next train must set its batch_size if inference is run with this model.
- */
- setBatchSize(X[0]->batch());
- }
-
- sharedConstTensors out;
- try {
- out = forwarding(X);
- /** Forward loss layer without label as well */
- out = std::static_pointer_cast<LossLayer>(layers[layers.size() - 1])
- ->forwarding(out);
- } catch (...) {
- ml_loge("Failed to inference Model");
- return out;
- }
- return out;
-}
-
-int NeuralNetwork::train(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- if (data_buffer == nullptr) {
- ml_loge("Cannot initialize the model without the data buffer.");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- status = setTrainConfig(values);
- NN_RETURN_STATUS();
-
- /** set batch size just before training */
- setBatchSize(batch_size);
-
- /** Setup data buffer properties */
- status = data_buffer->setClassNum(getOutputDimension()[0].width());
- NN_RETURN_STATUS();
-
- status = data_buffer->setFeatureSize(layers[0]->getInputDimension()[0]);
- NN_RETURN_STATUS();
-
- status = data_buffer->init();
- NN_RETURN_STATUS();
-
- return train_run();
-}
-
-/**
- * @brief Run NeuralNetwork train with callback function by user
- */
-int NeuralNetwork::train_run() {
- int status = ML_ERROR_NONE;
-
- for (epoch_idx = epoch_idx + 1; epoch_idx <= epochs; ++epoch_idx) {
- training.loss = 0.0f;
- status = data_buffer->run(nntrainer::BufferType::BUF_TRAIN);
- if (status != ML_ERROR_NONE) {
- data_buffer->clear(BufferType::BUF_TRAIN);
- return status;
- }
-
- if (data_buffer->getValidation()[(int)nntrainer::BufferType::BUF_TEST]) {
- status = data_buffer->run(nntrainer::BufferType::BUF_TEST);
- if (status != ML_ERROR_NONE) {
- data_buffer->clear(BufferType::BUF_TEST);
- return status;
- }
- }
-
- int count = 0;
-
- sharedTensor in = MAKE_SHARED_TENSOR(getInputDimension()[0]);
- sharedTensor label = MAKE_SHARED_TENSOR(getOutputDimension()[0]);
-
- while (true) {
- if (data_buffer->getDataFromBuffer(nntrainer::BufferType::BUF_TRAIN,
- in->getData(), label->getData())) {
- try {
- backwarding({in}, {label}, iter++);
- } catch (...) {
- data_buffer->clear(nntrainer::BufferType::BUF_TRAIN);
- ml_loge("Error: training error in #%d/%d.", epoch_idx, epochs);
- std::rethrow_exception(std::current_exception());
- }
- std::cout << "#" << epoch_idx << "/" << epochs;
- data_buffer->displayProgress(count++, nntrainer::BufferType::BUF_TRAIN,
- getLoss());
- training.loss += getLoss();
- } else {
- data_buffer->clear(nntrainer::BufferType::BUF_TRAIN);
- break;
- }
- }
-
- if (count == 0)
- throw std::runtime_error("No training data");
-
- training.loss /= count;
- saveModel();
-
- std::cout << "#" << epoch_idx << "/" << epochs
- << " - Training Loss: " << training.loss;
-
- if (data_buffer->getValidation()[(int)nntrainer::BufferType::BUF_VAL]) {
- int right = 0;
- validation.loss = 0.0f;
- unsigned int tcases = 0;
-
- status = data_buffer->run(nntrainer::BufferType::BUF_VAL);
- if (status != ML_ERROR_NONE) {
- data_buffer->clear(BufferType::BUF_VAL);
- return status;
- }
-
- while (true) {
- if (data_buffer->getDataFromBuffer(nntrainer::BufferType::BUF_VAL,
- in->getData(), label->getData())) {
- sharedConstTensors Y = forwarding({in}, {label});
- auto model_out = Y[0]->argmax();
- auto label_out = label->argmax();
- for (unsigned int b = 0; b < batch_size; b++) {
- if (model_out[b] == label_out[b])
- right++;
- }
- validation.loss += getLoss();
- tcases++;
- } else {
- data_buffer->clear(nntrainer::BufferType::BUF_VAL);
- break;
- }
- }
-
- if (tcases == 0) {
- ml_loge("Error : 0 test cases");
- status = ML_ERROR_INVALID_PARAMETER;
- return status;
- }
- validation.loss /= (float)(tcases);
- validation.accuracy = right / (float)(tcases * batch_size) * 100.0f;
- std::cout << " >> [ Accuracy: " << validation.accuracy
- << "% - Validation Loss : " << validation.loss << " ] ";
- }
- std::cout << std::endl;
- }
-
- return status;
-}
-
-int NeuralNetwork::isInitializable() {
- if (layers.empty()) {
- ml_loge("Layer is empty");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- Layer &l = *layers[0];
-
- /** Dimension of first layer must be known */
- if (l.getInputDimension().size() == 0) {
- ml_loge("InputDimension of first layer is not set");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- /** First layer cannot be activation, batch normalization or loss */
- switch (l.getType()) {
- case LayerType::LAYER_ACTIVATION:
- /// fallthrough intended
- case LayerType::LAYER_BN:
- /// fallthrough intended
- case LayerType::LAYER_LOSS:
- /// fallthrough intended
- ml_loge("%s cannot be the first layer, type: %d", l.getName().c_str(),
- static_cast<std::underlying_type<LayerType>::type>(l.getType()));
- return ML_ERROR_INVALID_PARAMETER;
- default:
- /// normal case
- break;
- }
-
- return ML_ERROR_NONE;
-}
-
-int NeuralNetwork::addLayer(NodeType layer) {
- int status = ML_ERROR_NONE;
-
- if (initialized) {
- return ML_ERROR_NOT_SUPPORTED;
- }
-
- /** Ensure that the layer has a name and is unique */
- ensureName(layer);
-
- /** Validate the layer to be added */
- status = layer->checkValidation();
- if (status != ML_ERROR_NONE) {
- ml_loge("layer(%s) validation failed.", layer->getName().c_str());
- return status;
- }
-
- /** Insert the layer to the graph */
- layers.push_back(layer);
-
- return status;
-}
-
-int NeuralNetwork::extendGraph(GraphType graph, std::string prefix) {
- if (initialized) {
- return ML_ERROR_NOT_SUPPORTED;
- }
-
- /** Insert the layer to the graph */
- for (auto layer : graph) {
- /**
- * Add prefix to the existing layer name,
- * and ensure it is unique in this new graph
- */
- ensureName(layer, prefix, true);
-
- layers.push_back(layer);
- }
-
- return ML_ERROR_NONE;
-}
-
-int NeuralNetwork::setOptimizer(
- std::shared_ptr<ml::train::Optimizer> optimizer) {
-
- if (optimizer->getType() == OptType::UNKNOWN)
- return ML_ERROR_INVALID_PARAMETER;
-
- if (initialized) {
- return ML_ERROR_NOT_SUPPORTED;
- }
-
- opt = std::static_pointer_cast<Optimizer>(optimizer);
-
- return ML_ERROR_NONE;
-}
-
-int NeuralNetwork::setDataBuffer(std::shared_ptr<DataBuffer> data_buffer) {
- this->data_buffer = data_buffer;
-
- return ML_ERROR_NONE;
-}
-
-void NeuralNetwork::ensureName(NodeType layer, const std::string &prefix,
- bool force_rename) {
- std::string orig_name = layer->getName();
- bool orig_name_empty = orig_name.empty();
- if (!orig_name_empty && !force_rename &&
- layer_names.end() == layer_names.find(orig_name))
- return;
-
- /** If just prefix with layer name makes it unique - directly set the name */
- if (!orig_name_empty) {
- std::string direct_name = prefix + orig_name;
- if (layer_names.find(direct_name) == layer_names.end()) {
- layer->setName(direct_name);
- return;
- }
- }
-
- std::set<std::string>::iterator iter;
- std::string name;
- if (orig_name_empty)
- orig_name = layer->getBaseName();
- std::string direct_name = prefix + orig_name;
-
- do {
- name = direct_name + std::to_string(def_name_count++);
- iter = layer_names.find(name);
- } while (iter != layer_names.end());
-
- layer->setName(name);
-}
-
-int NeuralNetwork::getLayer(const char *name,
- std::shared_ptr<ml::train::Layer> *layer) {
- std::shared_ptr<Layer> layer_;
- int ret = getLayer(name, &layer_);
- if (ret == ML_ERROR_NONE)
- *layer = layer_;
- return ret;
-}
-
-int NeuralNetwork::getLayer(const char *name, NodeType *layer) {
- int status = ML_ERROR_INVALID_PARAMETER;
- std::string name_str(name);
-
- for (auto iter = layers.begin(); iter != layers.end(); ++iter) {
- if ((*iter)->getName() == name_str) {
- *layer = *iter;
- return ML_ERROR_NONE;
- }
- }
-
- return status;
-}
-
-int NeuralNetwork::realizeActivationType(const ActivationType act) {
- unsigned int position = layers.end() - layers.begin() - 1;
- return realizeActivationType(act, position);
-}
-
-int NeuralNetwork::realizeActivationType(const ActivationType act,
- const unsigned int position) {
- if (act == ActivationType::ACT_NONE) {
- /// ActivationType::ACT_NONE does not need realization
- return ML_ERROR_NONE;
- }
-
- if (layers.empty()) {
- ml_loge("layer is empty");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- Layer ¤t = *layers[position];
- if (current.getType() == LayerType::LAYER_ACTIVATION) {
- ml_loge("It is not allowed to realize ativation layer, possibly layer is "
- "added right after activation");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- if (act == ActivationType::ACT_UNKNOWN) {
- ml_loge("cannot realize unknown activation type");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- std::shared_ptr<ActivationLayer> act_layer =
- std::make_shared<ActivationLayer>();
- ensureName(act_layer, current.getName());
- act_layer->setActivation(act);
-
- layers.insert(layers.begin() + position + 1, act_layer);
- return ML_ERROR_NONE;
-}
-
-int NeuralNetwork::realizeFlattenType(const unsigned int position) {
- if (layers.empty()) {
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- Layer ¤t = *layers[position];
- if (current.getType() == LayerType::LAYER_FLATTEN) {
- ml_loge(
- "It is not allowed to realize flatten layer, possibly flatten layer is "
- "added right after flatten");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- std::shared_ptr<FlattenLayer> flatten_layer =
- std::make_shared<FlattenLayer>();
-
- ensureName(flatten_layer, current.getName());
- layers.insert(layers.begin() + position + 1, flatten_layer);
- return ML_ERROR_NONE;
-}
-
-int NeuralNetwork::realizeFlattenType() {
- unsigned int position = layers.end() - layers.begin() - 1;
- return realizeFlattenType(position);
-}
-
-/**
- * @brief Set loss type for the neural network.
- */
-int NeuralNetwork::setLoss(LossType loss_type) {
- if (loss_type == LossType::LOSS_UNKNOWN)
- return ML_ERROR_INVALID_PARAMETER;
-
- this->loss_type = loss_type;
- return ML_ERROR_NONE;
-}
-
-void NeuralNetwork::printMetrics(std::ostream &out, unsigned int flags) {
- switch (flags) {
- case ML_TRAIN_SUMMARY_MODEL_TRAIN_LOSS:
- out << training.loss << std::endl;
- break;
-
- case ML_TRAIN_SUMMARY_MODEL_VALID_LOSS:
- out << validation.loss << std::endl;
- break;
-
- case ML_TRAIN_SUMMARY_MODEL_VALID_ACCURACY:
- out << validation.accuracy << std::endl;
- break;
-
- default:
- break;
- }
-}
-
-void NeuralNetwork::printPreset(std::ostream &out, unsigned int preset) {
- /** print neuralnet metrics */
- printMetrics(out, preset);
- if (preset > ML_TRAIN_SUMMARY_TENSOR)
- return;
-
- Layer::PrintPreset layer_preset = Layer::PrintPreset::PRINT_NONE;
-
- ///@todo match flags with preset
- unsigned int flags = PRINT_INST_INFO | PRINT_GRAPH_INFO | PRINT_PROP |
- PRINT_OPTIMIZER | PRINT_METRIC;
-
- switch (preset) {
- case ML_TRAIN_SUMMARY_TENSOR:
- layer_preset = Layer::PrintPreset::PRINT_ALL;
- break;
- case ML_TRAIN_SUMMARY_LAYER:
- layer_preset = initialized ? Layer::PrintPreset::PRINT_SUMMARY
- : Layer::PrintPreset::PRINT_SUMMARY_META;
- break;
- case ML_TRAIN_SUMMARY_MODEL:
- break;
- default:
- throw std::invalid_argument("given verbosity is invalid");
- }
-
- print(out, flags, layer_preset);
-}
-
-void NeuralNetwork::print(std::ostream &out, unsigned int flags,
- Layer::PrintPreset layerPrintPreset) {
- if (flags & PRINT_INST_INFO) {
- out << "===================";
- printInstance(out, this);
- }
-
- if (flags & PRINT_GRAPH_INFO) {
- out << "graph contains " << layers.size() << " operation nodes\n";
- /// @todo print graph info
- }
-
- if (flags & PRINT_PROP) {
- /// @todo print neuralnet property
- /// @todo print mode (if it is eval or training)
- }
-
- if (flags & PRINT_OPTIMIZER) {
- /// @todo print optimizer (with print optimizer prop)
- }
-
- if (flags & PRINT_METRIC) {
- /// @todo print metric (currently it is done at printPreset as a workaround)
- /// @todo print loss function when it is not initialized. (if it is
- /// initialized, loss layer will be printed)
- }
-
- if (layers.empty()) {
- out << "model is empty!" << std::endl;
- return;
- }
-
- /** print layer properties */
- for (auto &layer : layers)
- layer->printPreset(out, layerPrintPreset);
-
- /// @todo Add status to check neuralnet has been run. #290
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file nnstreamer_layer.cpp
- * @date 26 October 2020
- * @brief This is class to encapsulate nnstreamer as a layer of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- *
- * @todo: provide input/output dimensions to nnstreamer for certain frameworks
- * @todo: support transposing the data to support NCHW nntrainer data to NHWC
- * nnstreamer data
- */
-
-#include <lazy_tensor.h>
-#include <nnstreamer_layer.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int NNStreamerLayer::nnst_info_to_tensor_dim(ml_tensors_info_h &out_res,
- TensorDim &dim) {
- int status = ML_ERROR_NONE;
- unsigned int count;
- ml_tensor_type_e type;
- ml_tensor_dimension dim_;
-
- status = ml_tensors_info_get_count(out_res, &count);
- if (status != ML_ERROR_NONE)
- return status;
-
- if (count > 1)
- return ML_ERROR_NOT_SUPPORTED;
-
- status = ml_tensors_info_get_tensor_type(out_res, 0, &type);
- if (status != ML_ERROR_NONE)
- return status;
-
- if (type != ML_TENSOR_TYPE_FLOAT32)
- return ML_ERROR_NOT_SUPPORTED;
-
- if (ML_TENSOR_RANK_LIMIT > MAXDIM)
- return ML_ERROR_NOT_SUPPORTED;
-
- status = ml_tensors_info_get_tensor_dimension(out_res, 0, dim_);
- if (status != ML_ERROR_NONE)
- return status;
-
- for (size_t i = 0; i < MAXDIM; i++)
- dim.setTensorDim(i, dim_[i]);
-
- /* reverse the dimension as nnstreamer stores dimension in reverse way */
- dim.reverse();
-
- return status;
-}
-
-NNStreamerLayer::~NNStreamerLayer() { finalizeError(ML_ERROR_NONE); }
-
-int NNStreamerLayer::finalizeError(int status) {
- if (in_res) {
- ml_tensors_info_destroy(in_res);
- in_res = nullptr;
- }
-
- if (out_res) {
- ml_tensors_info_destroy(out_res);
- out_res = nullptr;
- }
-
- if (in_data_cont) {
- ml_tensors_data_destroy(in_data_cont);
- in_data_cont = nullptr;
- }
-
- if (out_data_cont) {
- ml_tensors_data_destroy(out_data_cont);
- out_data_cont = nullptr;
- }
-
- if (single) {
- ml_single_close(single);
- single = nullptr;
- }
-
- return status;
-}
-
-int NNStreamerLayer::initialize() {
- int status = ML_ERROR_NONE;
- TensorDim in_dim;
-
- status = ml_single_open(&single, modelfile.c_str(), NULL, NULL,
- ML_NNFW_TYPE_TENSORFLOW_LITE, ML_NNFW_HW_ANY);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- /* input tensor in filter */
- status = ml_single_get_input_info(single, &in_res);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- status = nnst_info_to_tensor_dim(in_res, in_dim);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- if (input_dim[0].getTensorDim(0) != 0 && input_dim[0] != in_dim) {
- ml_loge("Set tensor info does not match info from the framework");
- return finalizeError(ML_ERROR_INVALID_PARAMETER);
- } else {
- input_dim[0] = in_dim;
- }
-
- /* input tensor in filter */
- status = ml_single_get_output_info(single, &out_res);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- status = nnst_info_to_tensor_dim(out_res, output_dim[0]);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- /* generate input data container */
- status = ml_tensors_data_create(in_res, &in_data_cont);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- size_t in_data_size;
- status =
- ml_tensors_data_get_tensor_data(in_data_cont, 0, &in_data, &in_data_size);
- if (status != ML_ERROR_NONE)
- return finalizeError(status);
-
- if (in_data_size != input_dim[0].getDataLen() * sizeof(float))
- return finalizeError(ML_ERROR_INVALID_PARAMETER);
-
- return status;
-}
-
-void NNStreamerLayer::setTrainable(bool train) {
- if (train)
- throw exception::not_supported(
- "NNStreamer layer does not support training");
-
- Layer::setTrainable(false);
-}
-
-void NNStreamerLayer::setProperty(const PropertyType type,
- const std::string &value) {
- switch (type) {
- case PropertyType::modelfile: {
- if (!value.empty())
- modelfile = value;
- } break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-sharedConstTensors NNStreamerLayer::forwarding(sharedConstTensors in) {
- size_t data_size;
- input = *in[0];
-
- std::copy(input.getData(), input.getData() + input.length(),
- (float *)in_data);
-
- int status = ml_single_invoke(single, in_data_cont, &out_data_cont);
- if (status != ML_ERROR_NONE)
- throw std::runtime_error("Failed to forward nnstreamer backbone");
-
- hidden = Tensor(output_dim[0], static_cast<const float *>(out_data));
- status =
- ml_tensors_data_get_tensor_data(out_data_cont, 0, &out_data, &data_size);
- if (status != ML_ERROR_NONE) {
- ml_tensors_data_destroy(out_data_cont);
- out_data_cont = nullptr;
- throw std::runtime_error("Failed to forward nnstreamer backbone");
- }
-
- if (data_size != hidden.getSize()) {
- ml_tensors_data_destroy(out_data_cont);
- out_data_cont = nullptr;
- throw std::runtime_error("Output size mismatch from nnstreamer backbone.");
- }
-
- std::copy((float *)out_data, (float *)((char *)out_data + data_size),
- hidden.getData());
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-void NNStreamerLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<NNStreamerLayer> from =
- std::static_pointer_cast<NNStreamerLayer>(l);
- this->modelfile = from->modelfile;
-}
-
-sharedConstTensors NNStreamerLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- throw exception::not_supported(
- "Backwarding is not supported for nnstreamer layer");
-}
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * @file nntrainer_logger.cpp
- * @date 02 April 2020
- * @brief NNTrainer Logger
- * This allows to logging nntrainer logs.
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- */
-
-#include <cstring>
-#include <ctime>
-#include <iomanip>
-#include <mutex>
-#include <nntrainer_logger.h>
-#include <sstream>
-#include <stdarg.h>
-#include <stdexcept>
-
-namespace nntrainer {
-
-/**
- * @brief logfile name
- */
-const char *const Logger::logfile_name = "log_nntrainer_";
-
-/**
- * @brief instance for single logger
- */
-Logger *Logger::ainstance = nullptr;
-
-/**
- * @brief mutex for lock
- */
-std::mutex Logger::smutex;
-
-Logger &Logger::instance() {
- static Cleanup cleanup;
-
- std::lock_guard<std::mutex> guard(smutex);
- if (ainstance == nullptr)
- ainstance = new Logger();
- return *ainstance;
-}
-
-Logger::Cleanup::~Cleanup() {
- std::lock_guard<std::mutex> guard(Logger::smutex);
- delete Logger::ainstance;
- Logger::ainstance = nullptr;
-}
-
-Logger::~Logger() { outputstream.close(); }
-
-Logger::Logger() {
- struct tm lt;
- time_t t = time(0);
- struct tm *now = localtime_r(&t, <);
- std::stringstream ss;
- ss << logfile_name << std::dec << (now->tm_year + 1900) << std::setfill('0')
- << std::setw(2) << (now->tm_mon + 1) << std::setfill('0') << std::setw(2)
- << now->tm_mday << std::setfill('0') << std::setw(2) << now->tm_hour
- << std::setfill('0') << std::setw(2) << now->tm_min << std::setfill('0')
- << std::setw(2) << now->tm_sec << ".out";
- outputstream.open(ss.str(), std::ios_base::app);
- if (!outputstream.good()) {
- throw std::runtime_error("Unable to initialize the Logger!");
- }
-}
-
-void Logger::log(const std::string &message,
- const nntrainer_loglevel loglevel) {
- std::lock_guard<std::mutex> guard(smutex);
- time_t t = time(0);
- struct tm lt;
- struct tm *now = localtime_r(&t, <);
- std::stringstream ss;
- switch (loglevel) {
- case NNTRAINER_LOG_INFO:
- ss << "[NNTRAINER INFO ";
- break;
- case NNTRAINER_LOG_WARN:
- ss << "[NNTRAINER WARN ";
- break;
- case NNTRAINER_LOG_ERROR:
- ss << "[NNTRAINER ERROR ";
- break;
- case NNTRAINER_LOG_DEBUG:
- ss << "[NNTRAINER DEBUG ";
- break;
- default:
- break;
- }
-
- ss << std::dec << (now->tm_year + 1900) << '-' << std::setfill('0')
- << std::setw(2) << (now->tm_mon + 1) << '-' << std::setfill('0')
- << std::setw(2) << now->tm_mday << ' ' << std::setfill('0') << std::setw(2)
- << now->tm_hour << ':' << std::setfill('0') << std::setw(2) << now->tm_min
- << ':' << std::setfill('0') << std::setw(2) << now->tm_sec << ']';
-
- outputstream << ss.str() << " " << message << std::endl;
-}
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-void __nntrainer_log_print(nntrainer_loglevel loglevel,
- const std::string format_str, ...) {
- int final_n, n = ((int)format_str.size()) * 2;
- std::unique_ptr<char[]> formatted;
- va_list ap;
- while (1) {
- formatted.reset(new char[n]);
- std::strncpy(&formatted[0], format_str.c_str(), format_str.size());
- va_start(ap, format_str);
- final_n = vsnprintf(&formatted[0], n, format_str.c_str(), ap);
- va_end(ap);
- if (final_n < 0 || final_n >= n)
- n += abs(final_n - n + 1);
- else
- break;
- }
-
- std::string ss = std::string(formatted.get());
-
-#if defined(__LOGGING__)
- Logger::instance().log(ss, loglevel);
-#else
-
- switch (loglevel) {
- case NNTRAINER_LOG_ERROR:
- std::cerr << ss << std::endl;
- break;
- case NNTRAINER_LOG_INFO:
- case NNTRAINER_LOG_WARN:
- case NNTRAINER_LOG_DEBUG:
- std::cout << ss << std::endl;
- default:
- break;
- }
-
-#endif
-}
-
-#ifdef __cplusplus
-}
-#endif
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file optimizer.cpp
- * @date 08 April 2020
- * @brief This is Implementation of Optimizer class
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <cmath>
-#include <fstream>
-#include <iostream>
-
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <optimizer_internal.h>
-#include <parse_util.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int Optimizer::initialize(std::shared_ptr<Weight> weight_list,
- unsigned int num_weights, bool set_tensor) {
- return ML_ERROR_NONE;
-}
-
-double Optimizer::getLearningRate(int iteration) {
- double ll = learning_rate;
-
- if (decay_steps != 0) {
- ll = ll * pow(decay_rate, (iteration / (float)decay_steps));
- }
-
- return ll;
-}
-
-void Optimizer::apply_gradients(std::shared_ptr<Weight> weight_list,
- unsigned int num_weights, int iteration) {
-
- double ll = getLearningRate(iteration);
-
- int idx = 0;
- for (unsigned int i = 0; i < num_weights; ++i) {
- Weight &weight = weight_list.get()[i];
-
- if (!weight.getTrainable())
- continue;
-
- apply_gradient(weight, idx, ll, iteration);
- idx += 1;
- }
-}
-
-int Optimizer::setProperty(std::vector<std::string> values) {
- int status = ML_ERROR_NONE;
-
- for (unsigned int i = 0; i < values.size(); ++i) {
- std::string key;
- std::string value;
-
- status = getKeyValue(values[i], key, value);
- NN_RETURN_STATUS();
-
- unsigned int type = parseOptProperty(key);
-
- if (value.empty()) {
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- try {
- /// @note this calls derived setProperty if available
- setProperty(static_cast<PropertyType>(type), value);
- } catch (...) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
-
- try {
- checkValidation();
- } catch (...) {
- return ML_ERROR_INVALID_PARAMETER;
- }
- return status;
-}
-
-void Optimizer::checkValidation() {
- if (learning_rate <= 0.0f)
- throw std::invalid_argument("Learning rate must be positive");
-}
-
-void Optimizer::setProperty(const PropertyType type, const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::learning_rate:
- status = setFloat(learning_rate, value);
- break;
- case PropertyType::decay_steps:
- status = setUint(decay_steps, value);
- break;
- case PropertyType::decay_rate:
- status = setFloat(decay_rate, value);
- break;
- case PropertyType::continue_train:
- status = setBoolean(continue_train, value);
- break;
- default:
- ml_loge("Error: Unknown Optimizer Property Key");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
-
- throw_status(status);
-}
-
-void Optimizer::read(std::ifstream &file) {
- OptType loaded_type;
- file.read((char *)&loaded_type, sizeof(OptType));
-
- if (loaded_type >= OptType::UNKNOWN)
- throw std::runtime_error("Saved file has unknown optimizer");
-}
-
-void Optimizer::save(std::ofstream &file) {
- if (type >= OptType::UNKNOWN)
- throw std::runtime_error("Cannot save unknown optimizer");
-
- file.write((char *)&type, sizeof(OptType));
-}
-} // namespace nntrainer
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file optimizer_factory.cpp
- * @date 7 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the optimizer factory.
- */
-
-#include <adam.h>
-#include <optimizer_factory.h>
-#include <sgd.h>
-
-namespace nntrainer {
-
-/**
- * @brief Factory creator with copy constructor
- */
-std::unique_ptr<Optimizer> createOptimizer(OptType type, const Optimizer &opt) {
- switch (type) {
- case OptType::SGD:
- return std::make_unique<SGD>(static_cast<const SGD &>(opt));
- case OptType::ADAM:
- return std::make_unique<Adam>(static_cast<const Adam &>(opt));
- case OptType::UNKNOWN:
- /** fallthrough intended */
- default:
- throw std::invalid_argument("Unknown type for the optimizer");
- }
-}
-
-/**
- * @brief Factory creator with constructor
- */
-std::unique_ptr<Optimizer> createOptimizer(OptType type) {
- switch (type) {
- case OptType::SGD:
- return std::make_unique<SGD>();
- case OptType::ADAM:
- return std::make_unique<Adam>();
- case OptType::UNKNOWN:
- /** fallthrough intended */
- default:
- throw std::invalid_argument("Unknown type for the optimizer");
- }
-}
-
-} // namespace nntrainer
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file parse_util.cpp
- * @date 08 April 2020
- * @brief This is collection of math functions
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <array>
-#include <assert.h>
-#include <cstring>
-#include <iostream>
-#include <layer_internal.h>
-#include <neuralnet.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <optimizer_internal.h>
-#include <parse_util.h>
-#include <pooling2d_layer.h>
-#include <sstream>
-#include <string>
-
-#define NUM_SKIP_CHAR 3
-
-namespace nntrainer {
-
-int getKeyValue(std::string input_str, std::string &key, std::string &value) {
- int status = ML_ERROR_NONE;
- std::vector<std::string> list;
- std::regex words_regex("[^\\s=]+");
- input_str.erase(std::remove(input_str.begin(), input_str.end(), ' '),
- input_str.end());
- auto words_begin =
- std::sregex_iterator(input_str.begin(), input_str.end(), words_regex);
- auto words_end = std::sregex_iterator();
- int nwords = std::distance(words_begin, words_end);
- if (nwords != 2) {
- ml_loge("Error: input string must be 'key = value' format");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- for (std::sregex_iterator i = words_begin; i != words_end; ++i) {
- list.push_back((*i).str());
- }
-
- key = list[0];
- value = list[1];
- return status;
-}
-
-unsigned int parseType(std::string ll, InputType t) {
- unsigned int ret;
- unsigned int i;
- /**
- * @brief Optimizer String from configure file
- * "adam" : Adaptive Moment Estimation
- * "sgd" : Stochestic Gradient Descent
- */
- std::array<std::string, 2> optimizer_string = {"adam", "sgd"};
-
- /**
- * @brief Loss Function String from configure file
- * "mse" : Mean Squared Error
- * "caterogical" : Categorical Cross Entropy
- */
- std::array<std::string, 2> loss_string = {"mse", "cross"};
-
- /**
- * @brief Model Type String from configure file
- * "knn" : K Neearest Neighbor
- * "regression" : Logistic Regression
- * "neuralnet" : Neural Network
- */
- std::array<std::string, 3> model_type_string = {"knn", "regression",
- "neuralnet"};
-
- /**
- * @brief Activation Type String from configure file
- * "tanh" : tanh
- * "sigmoid" : sigmoid
- * "relu" : relu
- * "softmax" : softmax
- * "none" : none
- * "unknown" : unknown
- */
- std::array<std::string, 6> activation_string = {
- "tanh", "sigmoid", "relu", "softmax", "none", "unknown"};
-
- /**
- * @brief Layer Type String from configure file
- * "input" : Input Layer Object
- * "fully_conntected" : Fully Connected Layer Object
- * "batch_normalization" : Batch Normalization Layer Object
- * "conv2d" : Convolution 2D Layer Object
- * "pooling2d" : Pooling 2D Layer Object
- * "flatten" : Flatten Layer Object
- * "activation" : Activation Layer Object
- * "addition" : Addition Layer Object
- */
- std::array<std::string, 8> layer_string = {
- "input", "fully_connected", "batch_normalization", "conv2d",
- "pooling2d", "flatten", "activation", "addition"};
-
- /**
- * @brief Weight Initialization Type String from configure file
- * "zeros" : Zero Initialization
- * "ones" : One Initialization
- * "lecun_normal" : LeCun Normal Initialization
- * "lecun_uniform" : LeCun Uniform Initialization
- * "xavier_normal" : Xavier Normal Initialization
- * "xavier_uniform" : Xavier Uniform Initialization
- * "he_normal" : He Normal Initialization
- * "he_uniform" : He Uniform Initialization
- */
- std::array<std::string, 8> weight_ini_string = {
- "zeros", "ones", "lecun_normal", "lecun_uniform",
- "xavier_normal", "xavier_uniform", "he_normal", "he_uniform"};
-
- /**
- * @brief Weight Decay String from configure file
- * "L2Norm" : squared norm regularization
- * "Regression" : Regression
- */
- std::array<std::string, 2> weight_regularizer_string = {"l2norm",
- "regression"};
-
- /**
- * @brief Weight Decay String from configure file
- * "L2Norm" : squared norm regularization
- * "Regression" : Regression
- */
- std::array<std::string, 3> padding_string = {"full", "same", "valid"};
-
- /**
- * @brief Pooling String from configure file
- * "max" : Max Pooling
- * "average" : Average Pooling
- * "global_max" : Global Max Pooling
- * "global_average" : Global Average Pooling
- */
- std::array<std::string, 4> pooling_string = {"max", "average", "global_max",
- "global_average"};
-
- switch (t) {
- case TOKEN_OPT:
- for (i = 0; i < optimizer_string.size(); i++) {
- if (!strncasecmp(optimizer_string[i].c_str(), ll.c_str(),
- optimizer_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)OptType::UNKNOWN;
- break;
- case TOKEN_LOSS:
- for (i = 0; i < loss_string.size(); i++) {
- if (!strncasecmp(loss_string[i].c_str(), ll.c_str(),
- loss_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)LossType::LOSS_UNKNOWN;
- break;
- case TOKEN_MODEL:
- for (i = 0; i < model_type_string.size(); i++) {
- if (!strncasecmp(model_type_string[i].c_str(), ll.c_str(),
- model_type_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)NetType::UNKNOWN;
- break;
- case TOKEN_ACTI:
- for (i = 0; i < activation_string.size(); i++) {
- if (!strncasecmp(activation_string[i].c_str(), ll.c_str(),
- activation_string[i].size())) {
-
- return (i);
- }
- }
- ml_logw("Input activation %s cannot be identified. "
- "Moved to NO activation layer by default.",
- ll.c_str());
- ret = (unsigned int)ActivationType::ACT_UNKNOWN;
- break;
- case TOKEN_LAYER:
- for (i = 0; i < layer_string.size(); i++) {
- if (!strncasecmp(layer_string[i].c_str(), ll.c_str(),
- layer_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)LayerType::LAYER_UNKNOWN;
- break;
- case TOKEN_WEIGHT_INIT:
- for (i = 0; i < weight_ini_string.size(); i++) {
- if (!strncasecmp(weight_ini_string[i].c_str(), ll.c_str(),
- weight_ini_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)WeightInitializer::WEIGHT_UNKNOWN;
- break;
- case TOKEN_WEIGHT_REGULARIZER:
- for (i = 0; i < weight_regularizer_string.size(); i++) {
- if (!strncasecmp(weight_regularizer_string[i].c_str(), ll.c_str(),
- weight_regularizer_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)WeightRegularizerType::unknown;
- break;
- case TOKEN_PADDING:
- for (i = 0; i < padding_string.size(); i++) {
- if (!strncasecmp(padding_string[i].c_str(), ll.c_str(),
- padding_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)Pooling2DLayer::PaddingType::unknown;
- break;
- case TOKEN_POOLING:
- for (i = 0; i < pooling_string.size(); i++) {
- if (!strncasecmp(pooling_string[i].c_str(), ll.c_str(),
- pooling_string[i].size())) {
- return (i);
- }
- }
- ret = (unsigned int)Pooling2DLayer::PoolingType::unknown;
- break;
- case TOKEN_UNKNOWN:
- default:
- ml_loge("Error: unknown token cannot be parsed.");
- ret = 0;
- break;
- }
- return ret;
-}
-
-/**
- * @brief Layer Properties
- * input_shape = 0
- * normalization = 1
- * standardization = 2
- * activation = 3
- * epsilon = 4
- * weight_regularizer = 5
- * weight_regularizer_constant = 6
- * unit = 7
- * weight_initializer = 8
- * bias_initializer = 9
- * filters = 10
- * kernel_size = 11
- * stride = 12
- * padding = 13
- * pool_size = 14
- * pooling = 15
- * flatten = 16
- * name = 17
- * num_inputs = 18
- * num_outputs = 19
- * momentum = 20
- * moving_mean_initializer = 21
- * moving_variance_initializer = 22
- * gamma_initializer = 23
- * beta_initializer = 24
- * input_layers = 25
- *
- * InputLayer has 0, 1, 2, 3 properties.
- * FullyConnectedLayer has 1, 4, 6, 7, 8, 9 properties.
- * Conv2DLayer has 0, 1, 4, 6, 7, 9, 10, 11, 12, 13 properties.
- * Pooling2DLayer has 12, 13, 14, 15 properties.
- * BatchNormalizationLayer has 0, 1, 5, 6, 7 properties.
- */
-static std::array<std::string, 28> property_string = {
- "input_shape",
- "normalization",
- "standardization",
- "activation",
- "epsilon",
- "weight_regularizer",
- "weight_regularizer_constant",
- "unit",
- "weight_initializer",
- "bias_initializer",
- "filters",
- "kernel_size",
- "stride",
- "padding",
- "pool_size",
- "pooling",
- "flatten",
- "name",
- "num_inputs",
- "num_outputs",
- "momentum",
- "moving_mean_initializer",
- "moving_variance_initializer",
- "gamma_initializer",
- "beta_initializer",
- "modelfile",
- "input_layers",
- "unknown"};
-
-unsigned int parseLayerProperty(std::string property) {
- unsigned int i;
-
- for (i = 0; i < property_string.size(); i++) {
- unsigned int size = (property_string[i].size() > property.size())
- ? property_string[i].size()
- : property.size();
-
- if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
- return (i);
- }
- }
-
- return (unsigned int)Layer::PropertyType::unknown;
-}
-
-std::string propToStr(unsigned int type) { return property_string[type]; }
-
-unsigned int parseOptProperty(std::string property) {
- unsigned int i;
-
- /**
- * @brief Optimizer Properties
- * learning_rate = 0,
- * decay_rate = 1,
- * decay_steps = 2
- * beta1 = 3,
- * beta2 = 4,
- * epsilon = 5,
- * continue_train = 6,
- */
- std::array<std::string, 7> property_string = {
- "learning_rate", "decay_rate", "decay_steps", "beta1",
- "beta2", "epsilon", "continue_train"};
-
- for (i = 0; i < property_string.size(); i++) {
- unsigned int size = (property_string[i].size() > property.size())
- ? property_string[i].size()
- : property.size();
-
- if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
- return (i);
- }
- }
-
- return (unsigned int)Optimizer::PropertyType::unknown;
-}
-
-unsigned int parseNetProperty(std::string property) {
- unsigned int i;
-
- /**
- * @brief Network Properties
- * loss_val = 0,
- * loss = 1,
- * batch_size = 2,
- * epochs = 3,
- * save_path = 4
- */
- std::array<std::string, 5> property_string = {
- "loss_val", "loss", "batch_size", "epochs", "save_path"};
-
- for (i = 0; i < property_string.size(); i++) {
- unsigned int size = (property_string[i].size() > property.size())
- ? property_string[i].size()
- : property.size();
-
- if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
- return (i);
- }
- }
-
- return (unsigned int)NeuralNetwork::PropertyType::unknown;
-}
-
-unsigned int parseDataProperty(std::string property) {
- unsigned int i;
-
- /**
- * @brief Data Properties
- * train_data = 0,
- * val_data = 1,
- * test_data = 2,
- * label_data = 3,
- * buffer_size = 4
- */
- std::array<std::string, 5> property_string = {
- "train_data", "val_data", "test_data", "label_data", "buffer_size"};
-
- for (i = 0; i < property_string.size(); i++) {
- unsigned int size = (property_string[i].size() > property.size())
- ? property_string[i].size()
- : property.size();
-
- if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
- return (i);
- }
- }
-
- return (unsigned int)DataBuffer::PropertyType::unknown;
-}
-
-int setUint(unsigned int &val, const std::string &str) {
- int status = ML_ERROR_NONE;
- try {
- val = (unsigned int)std::stoul(str.c_str());
- } catch (...) {
- ml_loge("Error: Wrong Type. Must be int");
- status = ML_ERROR_INVALID_PARAMETER;
- }
-
- return status;
-}
-
-int setFloat(float &val, std::string str) {
- int status = ML_ERROR_NONE;
- try {
- val = std::stof(str.c_str());
- } catch (...) {
- ml_loge("Error: Wrong Type. Must be float");
- status = ML_ERROR_INVALID_PARAMETER;
- }
- return status;
-}
-
-int setDouble(double &val, std::string str) {
- int status = ML_ERROR_NONE;
- try {
- val = std::stod(str.c_str());
- } catch (...) {
- ml_loge("Error: Wrong Type. Must be double");
- status = ML_ERROR_INVALID_PARAMETER;
- }
-
- return status;
-}
-
-int setBoolean(bool &val, std::string str) {
- int status = ML_ERROR_NONE;
- std::string t = "true";
- std::string f = "false";
-
- if (!strncasecmp(str.c_str(), t.c_str(), t.size())) {
- val = true;
- } else if (!strncasecmp(str.c_str(), f.c_str(), f.size())) {
- val = false;
- } else {
- status = ML_ERROR_INVALID_PARAMETER;
- }
-
- return status;
-}
-
-int getValues(int n_str, std::string str, int *value) {
- int status = ML_ERROR_NONE;
- std::regex words_regex("[^\\s.,:;!?]+");
- str.erase(std::remove(str.begin(), str.end(), ' '), str.end());
- auto words_begin = std::sregex_iterator(str.begin(), str.end(), words_regex);
- auto words_end = std::sregex_iterator();
-
- int num = std::distance(words_begin, words_end);
- if (num != n_str) {
- ml_loge("Number of Data is not match");
- return ML_ERROR_INVALID_PARAMETER;
- }
- int cn = 0;
- for (std::sregex_iterator i = words_begin; i != words_end; ++i) {
- value[cn] = std::stoi((*i).str());
- cn++;
- }
- return status;
-}
-
-const char *getValues(std::vector<int> values, const char *delimiter) {
- std::stringstream vec_str;
-
- if (values.empty())
- return "unknown";
-
- std::copy(values.begin(), values.end() - 1,
- std::ostream_iterator<int>(vec_str, delimiter));
- vec_str << values.back();
-
- return std::move(vec_str.str().c_str());
-}
-
-std::vector<std::string> split(const std::string &s, std::regex ®) {
- std::vector<std::string> out;
- char char_to_remove[NUM_SKIP_CHAR] = {' ', '[', ']'};
- std::string str = s;
- for (unsigned int i = 0; i < NUM_SKIP_CHAR; ++i) {
- str.erase(std::remove(str.begin(), str.end(), char_to_remove[i]),
- str.end());
- }
- std::regex_token_iterator<std::string::iterator> end;
- std::regex_token_iterator<std::string::iterator> iter(str.begin(), str.end(),
- reg, -1);
-
- while (iter != end) {
- out.push_back(*iter);
- ++iter;
- }
- return out;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- *
- * @file pooling2d_layer.cpp
- * @date 12 June 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is 2 Dimensional Pooling Layer Class for Neural Network
- *
- */
-
-#include <cstring>
-#include <limits>
-
-#include <layer_internal.h>
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <pooling2d_layer.h>
-#include <util_func.h>
-
-namespace nntrainer {
-
-int Pooling2DLayer::initialize() {
- int status = ML_ERROR_NONE;
-
- if (input_dim.size() != 1 || output_dim.size() != 1) {
- throw std::invalid_argument("Convolution layer only takes one input");
- }
-
- TensorDim &in_dim = input_dim[0];
- TensorDim &out_dim = output_dim[0];
-
- if (in_dim.getDataLen() == 1) {
- ml_logw("Warning: the length of previous layer dimension is one");
- }
-
- out_dim.batch(in_dim.batch());
- out_dim.channel(in_dim.channel());
-
- if (pooling_type == PoolingType::max ||
- pooling_type == PoolingType::average) {
- out_dim.height(
- (in_dim.height() - pool_size[0] + 2 * padding[0]) / stride[0] + 1);
- out_dim.width((in_dim.width() - pool_size[1] + 2 * padding[1]) / stride[1] +
- 1);
- } else {
- out_dim.height(1);
- out_dim.width(1);
- }
-
- if (pooling_type == PoolingType::max) {
- max_idx.resize(out_dim.getDataLen());
- }
-
- if (pooling_type == PoolingType::global_max) {
- max_idx_global.resize(out_dim.getDataLen());
- }
-
- return status;
-}
-
-sharedConstTensors Pooling2DLayer::forwarding(sharedConstTensors in) {
- input = *in[0];
-
- TensorDim hidden_dim = output_dim[0];
- TensorDim &in_dim = input_dim[0];
-
- hidden = Tensor(hidden_dim);
- hidden.setZero();
-
- for (unsigned int b = 0; b < in_dim.batch(); ++b) {
- Tensor in_padded = zero_pad(b, input, padding.data());
- Tensor result = pooling2d(b, in_padded);
- memcpy(hidden.getAddress(b * hidden.getDim().getFeatureLen()),
- result.getData(), result.getDim().getDataLen() * sizeof(float));
- }
-
- return {MAKE_SHARED_TENSOR(hidden)};
-}
-
-sharedConstTensors Pooling2DLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- unsigned int batch = input_dim[0].batch();
- unsigned int channel = input_dim[0].channel();
- unsigned int height = input_dim[0].height();
- unsigned int width = input_dim[0].width();
- unsigned int p_height = pool_size[0];
- unsigned int p_width = pool_size[1];
- unsigned int p_size = p_height * p_width;
-
- unsigned int J, K;
- Tensor result = Tensor(input_dim[0]);
- result.setZero();
- float *out = result.getData();
- switch (pooling_type) {
- case PoolingType::max: {
- for (unsigned int i = 0; i < derivative[0]->getDim().getDataLen(); ++i) {
- out[max_idx[i]] += derivative[0]->getData()[i];
- }
- } break;
- case PoolingType::average: {
- for (unsigned int b = 0; b < batch; ++b) {
- for (unsigned int i = 0; i < channel; ++i) {
- J = 0;
- for (unsigned int j = 0; j <= height - p_height; j += stride[0]) {
- K = 0;
- for (unsigned int k = 0; k <= width - p_width; k += stride[1]) {
- float del =
- derivative[0]->getValue(b, i, J, K) / static_cast<float>(p_size);
- for (unsigned int pi = 0; pi < p_height; ++pi) {
- for (unsigned int pj = 0; pj < p_width; ++pj) {
- result.setValue(b, i, j + pi, k + pj,
- result.getValue(b, i, j + pi, k + pj) + del);
- }
- }
- K++;
- }
- J++;
- }
- }
- }
- } break;
- case PoolingType::global_max: {
- for (unsigned int i = 0; i < derivative[0]->getDim().getDataLen(); ++i) {
- float der = derivative[0]->getData()[i] / max_idx_global[i].size();
- for (unsigned int m = 0; m < max_idx_global[i].size(); m++) {
- out[max_idx_global[i][m]] += der;
- }
- }
- } break;
- case PoolingType::global_average: {
- unsigned int p_size = width * height;
- for (unsigned int b = 0; b < batch; ++b) {
- for (unsigned int i = 0; i < channel; ++i) {
- float del = derivative[0]->getValue(b, i, 0, 0) / (p_size);
- for (unsigned int j = 0; j < height; ++j) {
- for (unsigned int k = 0; k < width; ++k) {
- result.setValue(b, i, j, k, del);
- }
- }
- }
- }
-
- } break;
- default:
- throw std::runtime_error("Error: Unknown Pooling Type");
- }
- return {MAKE_SHARED_TENSOR(std::move(result))};
-}
-
-int Pooling2DLayer::setSize(int *size, PropertyType type) {
- int status = ML_ERROR_NONE;
- switch (type) {
- case PropertyType::pool_size:
- for (int i = 0; i < POOLING2D_DIM; ++i) {
- pool_size[i] = size[i];
- }
- break;
- case PropertyType::stride:
- for (int i = 0; i < POOLING2D_DIM; ++i) {
- stride[i] = size[i];
- }
- break;
- case PropertyType::padding:
- for (int i = 0; i < POOLING2D_DIM; ++i) {
- padding[i] = size[i];
- }
- break;
- default:
- ml_loge("Error: Unknown Layer Property type");
- status = ML_ERROR_INVALID_PARAMETER;
- break;
- }
- return status;
-}
-
-void Pooling2DLayer::setBatch(unsigned int batch) {
- Layer::setBatch(batch);
-
- if (pooling_type == PoolingType::max) {
- max_idx.resize(output_dim[0].getDataLen());
- } else if (pooling_type == PoolingType::global_max) {
- max_idx_global.resize(output_dim[0].getDataLen());
- }
-}
-
-void Pooling2DLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<Pooling2DLayer> from =
- std::static_pointer_cast<Pooling2DLayer>(l);
-
- this->pooling_type = from->pooling_type;
-
- for (unsigned int i = 0; i < POOLING2D_DIM; ++i) {
- this->pool_size[i] = from->pool_size[i];
- this->stride[i] = from->stride[i];
- this->padding[i] = from->padding[i];
- }
-}
-
-void Pooling2DLayer::setProperty(const PropertyType type,
- const std::string &value) {
- int status = ML_ERROR_NONE;
-
- switch (type) {
- case PropertyType::pooling:
- if (!value.empty()) {
- pooling_type = (PoolingType)parseType(value, TOKEN_POOLING);
- if (pooling_type == PoolingType::unknown) {
- throw std::invalid_argument("[Pooling2d_layer]: Unknown pooling type");
- }
- break;
- }
- case PropertyType::pool_size:
- if (!value.empty()) {
- status = getValues(POOLING2D_DIM, value, (int *)(pool_size.data()));
- throw_status(status);
- if (pool_size[0] == 0 || pool_size[1] == 0) {
- throw std::invalid_argument(
- "[Pooling2d_layer] pool_size must be greater than 0");
- }
- }
- break;
- case PropertyType::stride:
- if (!value.empty()) {
- status = getValues(POOLING2D_DIM, value, (int *)(stride.data()));
- throw_status(status);
- if (stride[0] == 0 || stride[1] == 0) {
- throw std::invalid_argument(
- "[Pooling2d_layer] stride must be greater than 0");
- }
- }
- break;
- case PropertyType::padding:
- if (!value.empty()) {
- status = getValues(POOLING2D_DIM, value, (int *)(padding.data()));
- throw_status(status);
- if ((int)padding[0] < 0 || (int)padding[1] < 0) {
- throw std::invalid_argument(
- "[Pooling2d_layer] padding must be greater than 0");
- }
- }
- break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-Tensor Pooling2DLayer::pooling2d(unsigned int batch, Tensor &in) {
- unsigned int channel = in.channel();
- unsigned int height = in.height();
- unsigned int width = in.width();
- unsigned int p_height = pool_size[0];
- unsigned int p_width = pool_size[1];
- TensorDim &out_dim = output_dim[0];
- unsigned int base_idx = batch * out_dim.getFeatureLen();
-
- Tensor output(out_dim.channel(), out_dim.height(), out_dim.width());
-
- unsigned int J, K;
- switch (pooling_type) {
- case PoolingType::max: {
- for (unsigned int i = 0; i < channel; ++i) {
- J = 0;
- for (unsigned int j = 0; j <= height - p_height; j += stride[0]) {
- K = 0;
- for (unsigned int k = 0; k <= width - p_width; k += stride[1]) {
- float max = std::numeric_limits<float>::lowest();
- for (unsigned int pi = 0; pi < p_height; ++pi) {
- for (unsigned int pj = 0; pj < p_width; ++pj) {
- float val = in.getValue(0, i, j + pi, k + pj);
- if (max < val) {
- max_idx[base_idx + i * out_dim.height() * out_dim.width() +
- J * out_dim.width() + K] =
- batch * input_dim[0].getFeatureLen() + i * height * width +
- (j + pi) * width + (k + pj);
- max = val;
- }
- }
- }
- output.setValue(0, i, J, K, max);
- K++;
- }
- J++;
- }
- }
- } break;
- case PoolingType::average: {
- unsigned int p_size = p_height * p_width;
- for (unsigned int i = 0; i < channel; ++i) {
- J = 0;
- for (unsigned int j = 0; j <= height - p_height; j += stride[0]) {
- K = 0;
- for (unsigned int k = 0; k <= width - p_width; k += stride[1]) {
- float sum = 0.0f;
- for (unsigned int pi = 0; pi < p_height; ++pi) {
- for (unsigned int pj = 0; pj < p_width; ++pj) {
- sum += in.getValue(0, i, j + pi, k + pj);
- }
- }
- sum = sum / static_cast<float>(p_size);
- output.setValue(0, i, J, K, sum);
- K++;
- }
- J++;
- }
- }
- } break;
- case PoolingType::global_max: {
- output.setZero();
- for (unsigned int i = 0; i < channel; ++i) {
- unsigned int idx =
- batch * input_dim[0].getFeatureLen() + i * height * width;
- float max = std::numeric_limits<float>::lowest();
- max_idx_global[base_idx + i].clear();
- for (unsigned int j = 0; j < height; ++j) {
- for (unsigned int k = 0; k < width; ++k) {
- float val = in.getValue(0, i, j, k);
- if (max <= val) {
- if (max < val)
- max_idx_global[base_idx + i].clear();
- max_idx_global[base_idx + i].push_back(idx + j * width + k);
- max = val;
- }
- }
- }
- output.setValue(0, i, 0, 0, max);
- }
- } break;
- case PoolingType::global_average: {
- output.setZero();
- Tensor sum_wh = in.chain().sum(3).sum(2).run();
- for (unsigned int i = 0; i < channel; ++i) {
- output.setValue(0, i, 0, 0,
- sum_wh.getValue(0, i, 0, 0) / (in.width() * in.height()));
- }
- } break;
- default:
- ml_loge("Error: Unknown Pooling Type");
- throw std::runtime_error("Error: Unknown Pooling Type");
- break;
- }
-
- return output;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file sgd.cpp
- * @date 6 October 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is the SGD optimizer.
- */
-
-#include <sgd.h>
-
-namespace nntrainer {
-
-void SGD::apply_gradient(Weight &weight, int tensor_idx, double updated_lr,
- int iteration) {
- Tensor &x = weight.getVariableRef();
- const Tensor &x_grad = weight.getGradientRef();
- x.add_i(x_grad, -updated_lr);
-}
-
-} // namespace nntrainer
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file tensor.cpp
- * @date 04 December 2019
- * @brief This is Tensor class for calculation
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <assert.h>
-#include <cmath>
-#include <cstring>
-#include <fstream>
-#include <iomanip>
-#include <iterator>
-#include <random>
-#include <regex>
-#include <sstream>
-#include <stdio.h>
-
-#include <blas_interface.h>
-#include <lazy_tensor.h>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <parse_util.h>
-#include <tensor.h>
-#include <util_func.h>
-
-#define transposeloop(cl, ci, cj, ck, sl, si, sj, sk) \
- do { \
- unsigned int i, j, k, l; \
- int inidx = 0, outidx = 0; \
- for (cl = 0; cl < sl; cl++) \
- for (ci = 0; ci < si; ci++) \
- for (cj = 0; cj < sj; cj++) \
- for (ck = 0; ck < sk; ck++) { \
- outidx = si * sj * sk * cl + sj * sk * ci + sk * cj + ck; \
- inidx = l * SI * SJ * SK + i * SJ * SK + j * SK + k; \
- outptr[outidx] = inptr[inidx]; \
- } \
- } while (0);
-
-/** do clone of this, perform the operation and return the output */
-#define CLONE_OP_I(op, ...) \
- do { \
- Tensor clone = this->clone(); \
- if (clone.op(__VA_ARGS__) != ML_ERROR_NONE) { \
- std::stringstream ss; \
- ss << "Error: op " << __func__ << " failed"; \
- throw std::runtime_error(ss.str()); \
- } \
- return clone; \
- } while (0);
-
-namespace nntrainer {
-
-static auto rng = [] {
- std::mt19937 rng;
- rng.seed(getSeed());
- return rng;
-}();
-
-Tensor::Tensor(const TensorDim &d, const float *buf) : Tensor() {
- if (d.getDataLen() != 0) {
- dim = d;
- strides = d.computeStrides();
- data = std::shared_ptr<float>(new float[d.getDataLen()],
- std::default_delete<float[]>());
-
- if (buf != nullptr) {
- float *data = getData();
- unsigned int len = length();
-
- scopy(len, buf, 1, data, 1);
- }
- }
-}
-
-bool Tensor::operator==(const Tensor &rhs) const {
- if (this->dim != rhs.dim)
- return false;
-
- size_t len = length();
-
- if (len != rhs.length())
- return false;
-
- const float *data = getData();
- const float *rdata = rhs.getData();
-
- for (size_t i = 0; i < len; ++i) {
- if (std::isnan(data[i]) || std::isnan(rdata[i]) ||
- std::fabs(data[i] - rdata[i]) > epsilon)
- return false;
- }
-
- return true;
-}
-
-float Tensor::getValue(unsigned int batch, unsigned int c, unsigned int h,
- unsigned int w) const {
- return getData()[getIndex(batch, c, h, w)];
-}
-
-void Tensor::setValue(unsigned int batch, unsigned int c, unsigned int h,
- unsigned int w, float value) {
- if (!is_contiguous) {
- throw std::runtime_error("cannot set value of non-contiguous tensor");
- }
-
- getData()[getIndex(batch, c, h, w)] = value;
-}
-
-template <typename T> void Tensor::setDist(T dist) {
- float *data = getData();
- unsigned int len = length();
- for (unsigned int i = 0; i < len; ++i) {
- data[i] = dist(rng);
- }
-}
-
-void Tensor::setRandNormal(float mean, float std) {
- setDist<std::normal_distribution<float>>(
- std::normal_distribution<float>(mean, std));
-}
-
-void Tensor::setRandUniform(float min, float max) {
- setDist<std::uniform_real_distribution<float>>(
- std::uniform_real_distribution<float>(min, max));
-}
-
-Tensor::Tensor(
- std::vector<std::vector<std::vector<std::vector<float>>>> const &d) {
-
- if (d.empty() || d[0].empty() || d[0][0].empty() || d[0][0][0].empty()) {
- throw std::out_of_range(
- "[Tensor] trying to initialize Tensor from empty vector");
- }
-
- dim.batch(d.size());
- dim.channel(d[0].size());
- dim.height(d[0][0].size());
- dim.width(d[0][0][0].size());
- strides = dim.computeStrides();
- data = std::shared_ptr<float>(new float[dim.getDataLen()],
- std::default_delete<float[]>());
- is_contiguous = true;
-
- for (unsigned int i = 0; i < dim.batch(); ++i)
- for (unsigned int j = 0; j < dim.channel(); ++j)
- for (unsigned int k = 0; k < dim.height(); ++k)
- for (unsigned int l = 0; l < dim.width(); ++l)
- this->setValue(i, j, k, l, d[i][j][k][l]);
-}
-
-int Tensor::multiply_i(float const &value) {
-
- float *data = getData();
- unsigned int len = length();
-
- sscal(len, value, data, 1);
- return ML_ERROR_NONE;
-}
-
-Tensor Tensor::multiply(float const &value) { CLONE_OP_I(multiply_i, value); }
-
-int Tensor::divide_i(float const &value) {
- if (value == 0.0f) {
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- return this->multiply_i(1.0f / value);
-}
-
-Tensor Tensor::divide(float const &value) {
- if (value == 0.0f) {
- throw std::runtime_error("Error: Divide by zero");
- }
-
- CLONE_OP_I(divide_i, value);
-}
-
-int Tensor::add_i(float const &value) {
- float *data = getData();
- unsigned int len = length();
-
- Tensor tmp(dim);
- tmp.setValue(value);
- saxpy(len, 1, tmp.getData(), 1, data, 1);
-
- return ML_ERROR_NONE;
-}
-
-Tensor Tensor::add(float const &value) { CLONE_OP_I(add_i, value); }
-
-/**
- * @struct External Loop Info for broadcasted info
- * @brief External Loop Info for broadcasted iteration. Please refer to
- * DISABLED_private_external_loop_n in unittest_nntrainer_tensor.
- * @note This should better be implemented in iterator fashion before used
- * extensively.
- */
-struct Tensor::BroadcastInfo {
-
- /**
- * @brief Construct a new External Loop Info object
- *
- */
- BroadcastInfo() : strides{0, 0, 0, 0} {}
-
- unsigned int buffer_size; /**< virtual size of the buffer */
- int buffer_axis; /**< the smallest axis that should be looped.
- -1 means no loop needed*/
- std::array<unsigned int, MAXDIM>
- strides; /**< modified strides for the loop */
-};
-
-/**
- * @brief Add Tensor Element by Element without mem copy
- * @param[in] m Tensor to be added
- * #retval #ML_ERROR_NONE Successful
- * #retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
- * TODO: add axis rather doing add over the last two dimensions always
- */
-int Tensor::add_i(Tensor const &m, float const alpha) {
- auto f = [&](const BroadcastInfo &e, float *buf, const float *m_buf) {
- saxpy(e.buffer_size, alpha, m_buf, e.strides[3], buf, strides[3]);
- };
-
- return operator_i(m, f);
-}
-
-Tensor Tensor::add(Tensor const &m, float const alpha) const {
- CLONE_OP_I(add_i, m, alpha);
-}
-
-int Tensor::subtract_i(Tensor const &m) { return add_i(m, -1); }
-
-Tensor Tensor::subtract(Tensor const &m) const { return add(m, -1); }
-
-int Tensor::subtract_i(float const &value) { return this->add_i(-value); }
-
-Tensor Tensor::subtract(float const &value) { return this->add(-value); }
-
-Tensor Tensor::pow(float exponent) const {
- return apply([=](float in) { return powf(in, exponent); });
-}
-
-int Tensor::operator_i(
- Tensor const &m,
- std::function<void(const BroadcastInfo &e, float *, const float *)> v_func) {
-
- BroadcastInfo e;
-
- /// shortcut to cover when dimension matches
- /// note that buffer_size, the last stride is only used in v_func but it might
- /// be changed
- if (dim == m.dim) {
- e.buffer_size = length();
- e.strides[3] = 1;
- v_func(e, getData(), m.getData());
- return ML_ERROR_NONE;
- }
-
- try {
- e = this->computeBroadcastInfo(m);
- } catch (std::exception &err) {
- ml_loge("%s %s", typeid(err).name(), err.what());
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- return operator_i_util(m, v_func, e);
-}
-
-int Tensor::operator_i_util(
- Tensor const &m,
- std::function<void(const BroadcastInfo &e, float *, const float *)> v_func,
- const BroadcastInfo &e, int cur_axis, unsigned int offset,
- unsigned int m_offset) {
-
- float *buf = this->getData();
- const float *m_buf = m.getData();
- int status = ML_ERROR_NONE;
-
- if (e.buffer_axis == cur_axis) {
- v_func(e, buf + offset, m_buf + m_offset);
- return ML_ERROR_NONE;
- }
-
- cur_axis++;
- for (unsigned int i = 0; i < dim.getTensorDim(cur_axis); ++i) {
- unsigned int next_offset = offset + i * strides[cur_axis];
- unsigned int next_m_offset = m_offset + i * e.strides[cur_axis];
- status =
- operator_i_util(m, v_func, e, cur_axis, next_offset, next_m_offset);
- if (status != ML_ERROR_NONE) {
- ml_loge("[operator_i] failed: %d", status);
- return status;
- }
- }
-
- return status;
-}
-
-int Tensor::multiply_i(Tensor const &m) {
- auto f = [&](const BroadcastInfo &e, float *buf, const float *m_buf) {
- for (unsigned int i = 0; i < e.buffer_size; ++i) {
- *buf *= *m_buf;
- buf += strides[3];
- m_buf += e.strides[3];
- }
- };
-
- return operator_i(m, f);
-}
-
-Tensor Tensor::multiply(Tensor const &m) const { CLONE_OP_I(multiply_i, m); }
-
-int Tensor::divide_i(Tensor const &m) {
- auto f = [&](const BroadcastInfo &e, float *buf, const float *m_buf) {
- for (unsigned int i = 0; i < e.buffer_size; ++i) {
- *buf /= *m_buf;
- buf += strides[3];
- m_buf += e.strides[3];
- }
- };
-
- return operator_i(m, f);
-}
-
-Tensor Tensor::divide(Tensor const &m) const { CLONE_OP_I(divide_i, m); }
-
-/**
- * This is to sum the Tensor data according to the dim.batch().
- * Therefore the result has M(dim.batch(), 1, 1, 1) dimension.
- */
-Tensor Tensor::sum_by_batch() {
- Tensor ret(dim.batch(), 1, 1, 1);
- unsigned int feat_len = dim.getFeatureLen();
- unsigned int batch = dim.batch();
-
- const float *data = getData();
- float *rdata = ret.getData();
-
- Tensor ones(1, 1, 1, feat_len);
- ones.setValue(1.0);
- sgemv(CblasRowMajor, CblasNoTrans, batch, feat_len, 1, data, feat_len,
- ones.getData(), 1, 0.0, rdata, 1);
-
- return ret;
-}
-
-/**
- * @brief Calculate sum according to the axis.
- */
-Tensor Tensor::sum(unsigned int axis, float alpha) const {
- Tensor ret;
-
- const float *data = getData();
-
- if (axis >= 4)
- throw std::out_of_range("Error: axis is invalid");
-
- if (dim.getDim()[axis] == 1 and alpha == 1.0)
- return this->clone();
-
- switch (axis) {
- case 0: {
- ret = Tensor(1, dim.channel(), dim.height(), dim.width());
- unsigned int feat_len = dim.getFeatureLen();
- unsigned int batch = dim.batch();
- Tensor ones(1, 1, 1, batch);
- ones.setValue(alpha);
- sgemv(CblasRowMajor, CblasTrans, batch, feat_len, 1, data, feat_len,
- ones.getData(), 1, 0.0, ret.getData(), 1);
- } break;
- case 1: {
- ret = Tensor(dim.batch(), 1, dim.height(), dim.width());
- unsigned int feat_len = dim.height() * dim.width();
- unsigned int channel = dim.channel();
- Tensor ones(1, 1, 1, channel);
- ones.setValue(alpha);
- float *rdata = ret.getData();
- for (unsigned int k = 0; k < dim.batch(); ++k) {
- sgemv(CblasRowMajor, CblasTrans, channel, feat_len, 1,
- &data[k * dim.getFeatureLen()], feat_len, ones.getData(), 1, 0.0,
- &rdata[k * feat_len], 1);
- }
- } break;
- case 2: {
- ret = Tensor(dim.batch(), dim.channel(), 1, dim.width());
- unsigned int width = dim.width();
- unsigned int height = dim.height();
- Tensor ones(1, 1, 1, height);
- ones.setValue(alpha);
- float *rdata = ret.getData();
- for (unsigned int k = 0; k < dim.batch(); ++k) {
- for (unsigned int c = 0; c < dim.channel(); ++c) {
- unsigned int idx =
- k * dim.getFeatureLen() + c * dim.width() * dim.height();
- unsigned int ridx = k * ret.dim.getFeatureLen() + c * dim.width();
- sgemv(CblasRowMajor, CblasTrans, height, width, 1, &data[idx], width,
- ones.getData(), 1, 0.0, &rdata[ridx], 1);
- }
- }
- } break;
- case 3: {
- ret = Tensor(dim.batch(), dim.channel(), dim.height(), 1);
- unsigned int m = ret.dim.getDataLen();
- unsigned int n = dim.width();
- Tensor ones(1, 1, 1, n);
- ones.setValue(alpha);
- sgemv(CblasRowMajor, CblasNoTrans, m, n, 1, data, n, ones.getData(), 1, 0.0,
- ret.getData(), 1);
- } break;
- default:
- throw std::out_of_range("Error: Dimension cannot exceed 3");
- }
- return ret;
-}
-
-Tensor Tensor::sum(const std::vector<unsigned int> &axes, float alpha) const {
- if (axes.empty())
- throw std::invalid_argument("empty axes given");
-
- Tensor ret = this->sum(axes[0], alpha);
-
- for (unsigned int i = 1; i < axes.size(); ++i)
- ret = ret.sum(axes[i]);
-
- return ret;
-}
-
-/**
- * @note: This dot product flattens the fist 3 axis for the purpose of
- * computation. So, while performing, these matrices are behaving as 2-D
- * matrices. The dimensions are restored while returning back the tensor.
- */
-Tensor Tensor::dot(Tensor const &m, bool trans, bool trans_m) const {
- if (m.dim.rank() > 2) {
- throw exception::not_supported("Error: support only for rank of dot "
- "matrix <= 2");
- }
- if (trans && dim.rank() > 2) {
- throw exception::not_supported("Error: support only for rank of dot "
- "matrix <= 2 with trans");
- }
-
- unsigned int dim1 = batch() * channel() * height();
- unsigned int dim2 = width();
- unsigned int mdim1 = m.batch() * m.channel() * m.height();
- unsigned int mdim2 = m.width();
- Tensor result;
-
- unsigned int M, N, K, lda, ldb, ldc;
-
- if (!trans && !trans_m) {
- if (dim2 != mdim1)
- throw std::runtime_error(
- "Error: incompatible dimensions for dot product");
- K = mdim1; /** == dim2 */
- N = mdim2;
- M = dim1;
- lda = K;
- ldb = N;
- result = Tensor(batch(), channel(), height(), mdim2);
-
- // We are not set zero the result because of performnace reason.
- // However, result is not initialized properly. There might include garbage
- // like nan. When we have to use this value as in C = alpha*A*B + beta*C,
- // then have to check gargabe data of C is not effect or not.
-
- } else if (!trans && trans_m) {
- if (dim2 != mdim2)
- throw std::runtime_error(
- "Error: incompatible dimensions for dot product");
- K = mdim2; /** == dim2 */
- N = mdim1;
- M = dim1;
- lda = K;
- ldb = K;
- result = Tensor(batch(), channel(), height(), mdim1);
- } else if (trans && !trans_m) {
- if (dim1 != mdim1)
- throw std::runtime_error(
- "Error: incompatible dimensions for dot product");
- K = mdim1; /** == dim1 */
- N = mdim2;
- M = dim2;
- lda = M;
- ldb = N;
- result = Tensor(1, 1, dim2, mdim2);
- } else {
- if (dim1 != mdim2)
- throw std::runtime_error(
- "Error: incompatible dimensions for dot product");
- K = mdim2; /** == dim1 */
- N = mdim1;
- M = dim2;
- lda = M;
- ldb = K;
- result = Tensor(1, 1, dim2, mdim1);
- }
- ldc = N;
-
- const float *data = getData();
- const float *mdata = m.getData();
- float *rdata = result.getData();
- const float alpha = 1.0f;
- const float beta = 0.0f;
-
- enum CBLAS_TRANSPOSE transA = trans ? CblasTrans : CblasNoTrans;
- enum CBLAS_TRANSPOSE transB = trans_m ? CblasTrans : CblasNoTrans;
- sgemm(CblasRowMajor, transA, transB, M, N, K, alpha, data, lda, mdata, ldb,
- beta, rdata, ldc);
-
- return result;
-}
-
-Tensor Tensor::transpose(std::string direction) const {
- unsigned int SL, SI, SJ, SK;
- int dir[MAXDIM - 1];
- unsigned int fromDim[4];
- const float *inptr;
- float *outptr;
-
- fromDim[0] = dim.batch();
- fromDim[1] = dim.channel();
- fromDim[2] = dim.height();
- fromDim[3] = dim.width();
-
- getValues(3, direction, dir);
- Tensor result(dim.batch(), fromDim[dir[0] + 1], fromDim[dir[1] + 1],
- fromDim[dir[2] + 1]);
-
- int indexI = dir[0];
- int indexJ = dir[1];
-
- SL = fromDim[0], SI = fromDim[1], SJ = fromDim[2], SK = fromDim[3];
-
- inptr = getData();
- outptr = result.getData();
-
- switch (indexI) {
- case 0:
- if (indexJ == 1) {
- transposeloop(l, i, j, k, SL, SI, SJ, SK);
- } else {
- transposeloop(l, i, k, j, SL, SI, SK, SJ);
- }
- break;
- case 1:
- if (indexJ == 0) {
- transposeloop(l, j, i, k, SL, SJ, SI, SK);
- } else {
- transposeloop(l, j, k, i, SL, SJ, SK, SI);
- }
- break;
- case 2:
- if (indexJ == 0) {
- transposeloop(l, k, i, j, SL, SK, SI, SJ);
- } else {
- transposeloop(l, k, j, i, SL, SK, SJ, SI);
- }
- break;
- }
-
- return result;
-}
-
-Tensor Tensor::apply(std::function<float(float)> f) const {
- Tensor result(dim.batch(), dim.channel(), dim.height(), dim.width());
- const float *data = getData();
- float *rdata = result.getData();
-
- std::transform(data, data + length(), rdata, f);
-
- return result;
-}
-
-Tensor Tensor::apply(std::function<Tensor(Tensor)> f) const { return f(*this); }
-
-void Tensor::print(std::ostream &out) const {
- printInstance(out, this);
- const float *data = getData();
-
- unsigned int len = length();
-
- out << dim;
-
- if (len > 100) {
- out << '[' << data[0] << ' ' << data[1] << ' ' << data[2] << " ... "
- << data[len - 3] << ' ' << data[len - 2] << ' ' << data[len - 1] << ']'
- << std::endl;
- return;
- }
-
- std::ios init(NULL);
- init.copyfmt(out);
- for (unsigned int k = 0; k < dim.batch(); k++) {
- for (unsigned int l = 0; l < dim.channel(); l++) {
- for (unsigned int i = 0; i < dim.height(); i++) {
- for (unsigned int j = 0; j < dim.width(); j++) {
- out << std::setw(10) << std::setprecision(10)
- << this->getValue(k, l, i, j) << " ";
- }
- out << std::endl;
- }
- out << std::endl;
- }
- out << "-------" << std::endl;
- }
- out.copyfmt(init);
-}
-
-std::ostream &operator<<(std::ostream &out, Tensor const &m) {
- m.print(out);
- return out;
-}
-
-float *Tensor::getAddress(unsigned int i) {
- if (i > this->dim.getDataLen()) {
- ml_loge("Error: Index out of bounds");
- return nullptr;
- }
-
- return &getData()[i];
-}
-
-const float *Tensor::getAddress(unsigned int i) const {
- if (i > this->dim.getDataLen()) {
- ml_loge("Error: Index out of bounds");
- return nullptr;
- }
-
- return &getData()[i];
-}
-
-void Tensor::copy(const Tensor &from) {
- // todo: enable copy to non-contiguous tensor
- if (!is_contiguous) {
- throw std::runtime_error("Cannot copy non-contiguous tensor");
- }
-
- Tensor t = Tensor(from.getDim(), from.getData());
- swap(t, *this);
-}
-
-Tensor Tensor::clone() const {
- Tensor t;
- t.copy(*this);
- return t;
-}
-
-void Tensor::reshape(const TensorDim &d) {
- if (d.getDataLen() != dim.getDataLen()) {
- throw std::invalid_argument("Error: reshape cannot change the tensor size");
- }
- dim = d;
- strides = d.computeStrides();
-}
-
-void Tensor::save(std::ofstream &file) {
- file.write((char *)getData(), getSize());
-}
-
-void Tensor::read(std::ifstream &file) {
- file.read((char *)getData(), getSize());
-}
-
-/**
- * @brief Calculate average value according to the axis.
- */
-Tensor Tensor::average(unsigned int axis) const {
- if (axis >= MAXDIM)
- throw std::out_of_range(
- "negative axis or axis more then MAXDIM is invalid");
-
- unsigned int axis_size = dim.getDim()[axis];
- if (axis_size == 1)
- return this->clone();
-
- return this->sum(axis, 1.0 / ((float)axis_size));
-}
-
-Tensor Tensor::average(const std::vector<unsigned int> &axes) const {
- if (axes.empty())
- return this->average();
-
- TensorDim ret_shape;
- for (const auto &idx : axes) {
- if (idx >= MAXDIM) {
- throw std::out_of_range("axis more then MAXDIM is invalid");
- }
- ret_shape.setTensorDim(idx, dim.getTensorDim(idx));
- }
-
- return this->sum(axes, 1.0 / (float)ret_shape.getDataLen());
-}
-
-/**
- * @brief Calculate average value according to the axis.
- */
-Tensor Tensor::average() const {
- Tensor result = *this;
- result.reshape({1, 1, 1, dim.getDataLen()});
- return result.average(3);
-}
-
-void Tensor::setValue(float val) {
- float *data = getData();
- std::fill(data, data + length(), val);
-}
-
-void Tensor::setZero() { setValue(0); }
-
-std::vector<unsigned int> Tensor::argmax() const {
- const float *data = getData();
- std::vector<unsigned int> result;
- unsigned int batch_size = batch();
- unsigned int feature_len = dim.getFeatureLen();
-
- result.reserve(batch_size);
-
- for (unsigned int b = 0; b < batch_size; b++) {
- auto max_iter =
- std::max_element(data + b * feature_len, data + (b + 1) * feature_len);
- result[b] = std::distance(data, max_iter);
- }
-
- return result;
-}
-
-float Tensor::l2norm() const {
- unsigned int len = length();
- const float *data = getData();
-
- return snrm2(len, data, 1);
-}
-
-Tensor Tensor::normalization() const {
- const float *data = getData();
-
- auto bounds = std::minmax_element(data, data + length());
- const float min = *bounds.first;
- const float max = *bounds.second;
-
- if (max == min) {
- return this->subtract(*this);
- }
- return this->chain().subtract_i(min).divide_i(max - min).run();
-}
-
-LazyTensor Tensor::chain() const { return LazyTensor(*this); }
-
-Tensor Tensor::standardization() const {
- Tensor result(dim);
-
- const float *data = getData();
- float *rdata = result.getData();
-
- for (unsigned int k = 0; k < dim.batch(); ++k) {
- int K = k * dim.getFeatureLen();
- float mean;
- float mean_tmp = 0.0f;
- float std_tmp = 0.0f;
- float std_dev = 0.0f;
-
- for (unsigned int l = 0; l < dim.channel(); ++l) {
- unsigned int L = K + l * dim.height() * dim.width();
- for (unsigned int i = 0; i < dim.height(); ++i) {
- unsigned int I = L + i * dim.width();
- for (unsigned int j = 0; j < dim.width(); ++j) {
- unsigned int J = I + j;
- mean_tmp += data[J];
- }
- }
- }
-
- mean = mean_tmp / (this->dim.getFeatureLen());
-
- for (unsigned int l = 0; l < dim.channel(); ++l) {
- unsigned int L = K + l * dim.height() * dim.width();
- for (unsigned int i = 0; i < dim.height(); ++i) {
- unsigned int I = L + i * dim.width();
- for (unsigned int j = 0; j < dim.width(); ++j) {
- unsigned int J = I + j;
- std_tmp += (data[J] - mean) * (data[J] - mean);
- }
- }
- }
- std_dev = sqrt(std_tmp) / (this->dim.getFeatureLen());
-
- for (unsigned int l = 0; l < dim.channel(); ++l) {
- unsigned int L = K + l * dim.height() * dim.width();
- for (unsigned int i = 0; i < dim.height(); ++i) {
- unsigned int I = L + i * dim.width();
- for (unsigned int j = 0; j < dim.width(); ++j) {
- unsigned int J = I + j;
- rdata[J] = (data[J] - mean) / std_dev;
- }
- }
- }
- }
-
- return result;
-}
-
-Tensor::BroadcastInfo Tensor::computeBroadcastInfo(const Tensor &m) {
- if (m.length() > this->length())
- throw exception::not_supported("broadcasting *this is not supported");
-
- const TensorDim m_dim = m.getDim();
-
- BroadcastInfo e;
-
- /// checking if given Tensor's can be broadcasted
- for (unsigned int i = 0; i < MAXDIM; ++i) {
- if (dim.getTensorDim(i) == m_dim.getTensorDim(i)) {
- e.strides[i] = m.strides[i];
- continue;
- }
-
- /// If given dimension is 1, it could be reuesed, the stride remaining 0
- /// Need to check if dim[i] == 1 && m_dim[i] == 1 first though
- /// If so, strides should not change
- if (m_dim.getTensorDim(i) == 1) {
- continue;
- }
-
- std::stringstream ss;
- ss << "[computeBroadcastInfo] broadcasting only allowed for"
- "dimension value of 1 \n"
- << "this: " << dim << "target: " << m_dim;
- throw std::invalid_argument(ss.str().c_str());
- }
-
- /// calculate inner loop size
- e.buffer_size = 1;
- e.buffer_axis = -1;
- e.strides[3] = m.strides[3];
-
- /// initiate buffer info with matching dimension strategy
- for (int axis = 3; axis >= 0; --axis) {
- if (dim.getTensorDim(axis) != m_dim.getTensorDim(axis)) {
- e.buffer_axis = axis;
- break;
- }
-
- e.buffer_size *= dim.getTensorDim(axis);
- }
-
- /// check strategy that uses consecutive ones
- if (m_dim.getTensorDim(3) == 1) {
- unsigned int inner_loop_size = 1;
- int axis;
- for (axis = 3; axis >= 0; --axis) {
- if (m_dim.getTensorDim(axis) != 1) {
- break;
- }
-
- inner_loop_size *= dim.getTensorDim(axis);
- }
-
- /// if consecutive-one strategy has bigger chunk size, replace the
- /// information
- if (inner_loop_size > e.buffer_size) {
- e.buffer_axis = axis;
- e.buffer_size = inner_loop_size;
- e.strides[3] = 0;
- }
- }
-
- return e;
-}
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
- */
-/**
- * @file tensor_dim.cpp
- * @date 22 May 2020
- * @brief This is Tensor Dimension Class
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <cstring>
-#include <regex>
-#include <sstream>
-#include <stdio.h>
-
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <tensor_dim.h>
-
-namespace nntrainer {
-
-TensorDim::TensorDim(const std::string &shape) : TensorDim() {
- if (setTensorDim(shape) != ML_ERROR_NONE) {
- throw std::invalid_argument("[TensorDim] Setting TensorDim failed");
- }
-}
-
-TensorDim &TensorDim::operator=(const TensorDim &rhs) {
- using std::swap;
-
- TensorDim tmp(rhs.batch(), rhs.channel(), rhs.height(), rhs.width());
- swap(*this, tmp);
- return *this;
-}
-
-TensorDim &TensorDim::operator=(TensorDim &&rhs) noexcept {
- using std::swap;
-
- swap(*this, rhs);
- return *this;
-}
-
-void TensorDim::resetLen() {
- feature_len = dim[1] * dim[2] * dim[3];
- len = dim[0] * feature_len;
-}
-
-const unsigned int TensorDim::getTensorDim(unsigned int idx) const {
- if (idx >= MAXDIM)
- throw std::invalid_argument(
- "[TensorDim] Tensor Dimension index should be between 0 and 4");
-
- return dim[idx];
-}
-
-void TensorDim::setTensorDim(unsigned int idx, unsigned int value) {
- if (idx >= MAXDIM)
- throw std::out_of_range(
- "[TensorDim] Tensor Dimension index should be between 0 and 4");
-
- if (value <= 0)
- throw std::invalid_argument(
- "[TensorDim] Trying to assign value <=0 to tensor dim");
-
- if (len == 0) {
- for (size_t i = 0; i < MAXDIM; ++i) {
- dim[i] = 1;
- }
- }
-
- dim[idx] = value;
- resetLen();
-}
-
-int TensorDim::setTensorDim(const std::string &input_shape) {
- int status = ML_ERROR_NONE;
- std::regex words_regex("[^\\s.,:;!?]+");
- auto words_begin =
- std::sregex_iterator(input_shape.begin(), input_shape.end(), words_regex);
- auto words_end = std::sregex_iterator();
- int cur_dim = std::distance(words_begin, words_end);
- if (cur_dim <= 0 || (size_t)cur_dim > MAXDIM) {
- ml_loge("Tensor Dimension should be between 1 and 4");
- return ML_ERROR_INVALID_PARAMETER;
- }
- int cn = 0;
- for (std::sregex_iterator i = words_begin; i != words_end; ++i, ++cn) {
- setTensorDim(MAXDIM - cur_dim + cn, std::stoul((*i).str()));
- }
-
- return status;
-}
-
-bool TensorDim::operator==(const TensorDim &rhs) const {
- for (size_t i = 0; i < MAXDIM; ++i) {
- if (this->dim[i] != rhs.dim[i]) {
- return false;
- }
- }
-
- return true;
-}
-
-unsigned int TensorDim::rank() const {
- unsigned int rank = 0;
- for (unsigned int i = 0; i < MAXDIM; i++) {
- if (dim[i] > 1)
- rank += 1;
- }
- return rank;
-}
-
-unsigned int &TensorDim::operator[](const unsigned int index) {
- if (index >= MAXDIM)
- throw std::out_of_range(
- "[TensorDim] Tensor Dimension index should be between 0 and 4");
- return dim[index];
-}
-
-const unsigned int &TensorDim::operator[](const unsigned int index) const {
- if (index >= MAXDIM)
- throw std::out_of_range(
- "[TensorDim] Tensor Dimension index should be between 0 and 4");
- return dim[index];
-}
-
-void TensorDim::reverse() { std::reverse(dim, dim + MAXDIM); }
-
-std::ostream &operator<<(std::ostream &out, TensorDim const &d) {
- out << "Shape: " << d.batch() << ":" << d.channel() << ":" << d.height()
- << ":" << d.width() << std::endl;
- return out;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file tflite_layer.cpp
- * @date 26 October 2020
- * @brief This is class to encapsulate tflite as a layer of Neural Network
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- */
-
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <tflite_layer.h>
-
-namespace nntrainer {
-
-void TfLiteLayer::setDimensions(const std::vector<int> &tensor_idx_list,
- std::vector<TensorDim> &dim, bool is_output) {
- unsigned int num_tensors = tensor_idx_list.size();
- dim.resize(num_tensors);
-
- for (unsigned int i = 0; i < num_tensors; i++) {
- unsigned int tensor_idx = tensor_idx_list[i];
- if (is_output && interpreter->tensor(tensor_idx)->type != kTfLiteFloat32)
- throw exception::not_supported(
- "Data type other than float32 not supported");
-
- unsigned int num_dims = interpreter->tensor(tensor_idx)->dims->size;
- if (num_dims > MAXDIM)
- throw exception::not_supported("Number of dimensions exceed the support");
-
- /** This puts the unused dimensions to the outer dimensions */
- for (size_t dim_idx = 0; dim_idx < num_dims; dim_idx++)
- dim[i].setTensorDim(
- MAXDIM - dim_idx - 1,
- interpreter->tensor(tensor_idx)->dims->data[num_dims - dim_idx - 1]);
- }
-}
-
-int TfLiteLayer::initialize() {
- tflite::ops::builtin::BuiltinOpResolver resolver;
-
- model = tflite::FlatBufferModel::BuildFromFile(modelfile.c_str());
- if (!model)
- return ML_ERROR_INVALID_PARAMETER;
-
- tflite::InterpreterBuilder(*model.get(), resolver)(&interpreter);
- if (!interpreter)
- return ML_ERROR_INVALID_PARAMETER;
-
- if (interpreter->AllocateTensors() != kTfLiteOk)
- throw std::runtime_error("Failed to allocate tensors!");
-
- std::vector<TensorDim> dims;
- setDimensions(interpreter->inputs(), dims, false);
- setDimensions(interpreter->outputs(), output_dim, true);
-
- if (input_dim.size() && input_dim[0].getTensorDim(0) != 0) {
- if (dims.size() != input_dim.size())
- throw std::invalid_argument(
- "Provided number of input dimensions mismatch");
-
- for (size_t idx = 0; idx < dims.size(); idx++) {
- if (dims[idx] != input_dim[idx])
- throw std::invalid_argument("Input dimensions mismatch");
- }
- } else {
- input_dim.resize(dims.size());
- std::copy(dims.begin(), dims.end(), input_dim.begin());
- }
-
- return ML_ERROR_NONE;
-}
-
-void TfLiteLayer::setTrainable(bool train) {
- if (train)
- throw exception::not_supported("TfLite layer does not support training");
-
- Layer::setTrainable(false);
-}
-
-void TfLiteLayer::setProperty(const PropertyType type,
- const std::string &value) {
- switch (type) {
- case PropertyType::modelfile: {
- if (!value.empty())
- modelfile = value;
- } break;
- default:
- Layer::setProperty(type, value);
- break;
- }
-}
-
-sharedConstTensors TfLiteLayer::forwarding(sharedConstTensors in) {
-#ifdef DEBUG
- std::vector<TensorDim> dims;
- if (in.size() != input_dim.size())
- throw std::invalid_argument("Provided number of input dimensions mismatch");
-
- for (int idx = 0; idx < dims.size(); idx++) {
- if (in[idx].getDim() != input_dim[idx])
- throw std::invalid_argument("Input dimensions mismatch");
- }
-#endif
-
- sharedConstTensors out;
-
- auto in_indices = interpreter->inputs();
- for (size_t idx = 0; idx < in.size(); idx++)
- interpreter->tensor(in_indices[idx])->data.raw = (char *)in[idx]->getData();
-
- auto out_indices = interpreter->outputs();
- out.resize(out_indices.size());
- for (size_t idx = 0; idx < out_indices.size(); idx++) {
- out[idx] = MAKE_SHARED_TENSOR(output_dim[idx]);
- interpreter->tensor(out_indices[idx])->data.raw =
- (char *)out[idx]->getData();
- }
-
- int status = interpreter->Invoke();
- if (status != kTfLiteOk)
- throw std::runtime_error("Invoke failed");
-
- hidden = *out[0];
-
- return out;
-}
-
-void TfLiteLayer::copy(std::shared_ptr<Layer> l) {
- Layer::copy(l);
-
- std::shared_ptr<TfLiteLayer> from = std::static_pointer_cast<TfLiteLayer>(l);
- this->modelfile = from->modelfile;
-}
-
-sharedConstTensors TfLiteLayer::backwarding(sharedConstTensors derivative,
- int iteration) {
- throw exception::not_supported(
- "Backwarding is not supported for tflite layer");
-}
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * @file util_func.cpp
- * @date 08 April 2020
- * @brief This is collection of math functions
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <cmath>
-#include <fstream>
-#include <random>
-
-#include <util_func.h>
-
-namespace nntrainer {
-
-static auto rng = [] {
- std::mt19937 rng;
- rng.seed(getSeed());
- return rng;
-}();
-static std::uniform_real_distribution<float> dist(-0.5, 0.5);
-
-unsigned int getSeed() { return 0; }
-
-float random() { return dist(rng); }
-
-float sqrtFloat(float x) { return sqrt(x); };
-
-double sqrtDouble(double x) { return sqrt(x); };
-
-float logFloat(float x) { return log(x + 1.0e-20); }
-
-float exp_util(float x) { return exp(x); }
-
-// This is 2D zero pad
-// TODO : Optimize for multi dimention padding
-Tensor zero_pad(int batch, Tensor const &in, unsigned int const *padding) {
- unsigned int c = in.channel();
- unsigned int h = in.height();
- unsigned int w = in.width();
-
- unsigned int height_p = h + padding[0] * 2;
- unsigned int width_p = w + padding[1] * 2;
-
- unsigned int height_p_h = h + padding[0];
- unsigned int width_p_h = w + padding[1];
-
- Tensor output(1, c, height_p, width_p);
- output.setZero();
-
- for (unsigned int j = 0; j < c; ++j) {
- for (unsigned int k = 0; k < padding[0]; ++k) {
- for (unsigned int l = 0; l < width_p; ++l) {
- output.setValue(0, j, k, l, 0.0f);
- output.setValue(0, j, k + height_p_h, l, 0.0f);
- }
- }
-
- for (unsigned int l = 0; l < padding[1]; ++l) {
- for (unsigned int k = padding[0]; k < h; ++k) {
- output.setValue(0, j, k, l, 0.0f);
- output.setValue(0, j, k, l + width_p_h, 0.0f);
- }
- }
- }
-
- for (unsigned int j = 0; j < c; ++j) {
- for (unsigned int k = 0; k < h; ++k) {
- for (unsigned int l = 0; l < w; ++l) {
- output.setValue(0, j, k + padding[0], l + padding[1],
- in.getValue(batch, j, k, l));
- }
- }
- }
-
- return output;
-}
-
-// This is strip pad and return original tensor
-Tensor strip_pad(Tensor const &in, unsigned int const *padding) {
- Tensor output(in.batch(), in.channel(), in.height() - padding[0] * 2,
- in.width() - padding[1] * 2);
- output.setZero();
-
- for (unsigned int i = 0; i < in.batch(); ++i) {
- for (unsigned int j = 0; j < in.channel(); ++j) {
- for (unsigned int k = 0; k < output.height(); ++k) {
- for (unsigned int l = 0; l < output.width(); ++l) {
- output.setValue(i, j, k, l,
- in.getValue(i, j, k + padding[0], l + padding[1]));
- }
- }
- }
- }
- return output;
-}
-
-Tensor rotate_180(Tensor in) {
- Tensor output(in.getDim());
- output.setZero();
- for (unsigned int i = 0; i < in.batch(); ++i) {
- for (unsigned int j = 0; j < in.channel(); ++j) {
- for (unsigned int k = 0; k < in.height(); ++k) {
- for (unsigned int l = 0; l < in.width(); ++l) {
- output.setValue(
- i, j, k, l,
- in.getValue(i, j, (in.height() - k - 1), (in.width() - l - 1)));
- }
- }
- }
- }
- return output;
-}
-
-bool isFileExist(std::string file_name) {
- std::ifstream infile(file_name);
- return infile.good();
-}
-
-} // namespace nntrainer
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0-only
-/**
- * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
- *
- * @file weight.cpp
- * @date 22 September 2020
- * @see https://github.com/nnstreamer/nntrainer
- * @author Parichay Kapoor <pk.kapoor@samsung.com>
- * @bug No known bugs except for NYI items
- * @brief This is Weight Class for Neural Network
- *
- */
-
-#include <util_func.h>
-#include <weight.h>
-
-namespace nntrainer {
-
-Weight::Weight(const Weight &rhs) :
- initializer(rhs.initializer),
- trainable(rhs.trainable),
- name(rhs.name) {
- var = rhs.var.clone();
- grad = rhs.grad.clone();
-}
-
-Weight &Weight::operator=(const Weight &rhs) {
- Weight temp(rhs);
- swap(temp, *this);
- return *this;
-}
-
-Weight::Weight(const TensorDim &dim, const WeightInitializer init, bool train,
- std::string name) :
- initializer(init),
- trainable(train),
- name(name) {
- if (initializer == WeightInitializer::WEIGHT_UNKNOWN)
- throw std::invalid_argument("Weight initializer unknown");
-
- initializeVar(dim);
- if (trainable) {
- grad = Tensor(dim);
- grad.setZero();
- } else
- grad = Tensor();
-}
-
-void Weight::initializeVar(const TensorDim &dim) {
- var = Tensor(dim);
- switch (initializer) {
- case WeightInitializer::WEIGHT_ZEROS:
- var.setZero();
- break;
- case WeightInitializer::WEIGHT_ONES:
- var.setValue(1.0f);
- break;
- case WeightInitializer::WEIGHT_LECUN_NORMAL:
- var.setRandNormal(0.0f, sqrtFloat(1.0f / dim.height()));
- break;
- case WeightInitializer::WEIGHT_XAVIER_NORMAL:
- var.setRandNormal(0.0f, sqrtFloat(2.0f / (dim.width() + dim.height())));
- break;
- case WeightInitializer::WEIGHT_HE_NORMAL:
- var.setRandNormal(0.0f, sqrtFloat(2.0f / (dim.height())));
- break;
- case WeightInitializer::WEIGHT_LECUN_UNIFORM:
- var.setRandUniform(-1.0f * sqrtFloat(1.0f / dim.height()),
- sqrtFloat(1.0f / dim.height()));
- break;
- case WeightInitializer::WEIGHT_XAVIER_UNIFORM:
- var.setRandUniform(-1.0f * sqrtFloat(6.0f / (dim.height() + dim.width())),
- sqrtFloat(6.0 / (dim.height() + dim.width())));
- break;
- case WeightInitializer::WEIGHT_HE_UNIFORM:
- var.setRandUniform(-1.0f * sqrtFloat(6.0f / (dim.height())),
- sqrtFloat(6.0 / (dim.height())));
- break;
- default:
- break;
- }
-}
-
-} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file blas_interface.cpp
+ * @date 28 Aug 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is dummy header for blas support
+ *
+ */
+
+#include <blas_interface.h>
+#include <nntrainer_error.h>
+
+#include <cmath>
+
+#define sgemv_loop(ci, cj, cM, cN) \
+ do { \
+ double y0; \
+ unsigned int i, j; \
+ for (ci = 0; ci != cM; ci++) { \
+ y0 = Y[ci * incy] * beta; \
+ for (cj = 0; cj != cN; cj++) \
+ y0 += A[i + j * lda] * X[cj * incx]; \
+ Y[ci * incy] = y0; \
+ } \
+ } while (0);
+
+namespace nntrainer {
+
+#ifndef USE_BLAS
+
+static void saxpy_raw(const unsigned int N, const float alpha, const float *X,
+ const int incX, float *Y, const int incY) {
+ if (incX <= 0 or incY <= 0)
+ throw std::invalid_argument(
+ "Error: negative inc not supported without cblas");
+ for (unsigned int i = 0; i < N; ++i)
+ Y[i * incY] = Y[i * incY] + X[i * incX] * alpha;
+}
+
+static void sgemv_raw(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA,
+ const unsigned int M, const unsigned int N,
+ const float alpha, const float *A, const unsigned int lda,
+ const float *X, const int incX, const float beta,
+ float *Y, const int incY) {
+
+ unsigned int incy = abs(incY);
+ unsigned int incx = abs(incX);
+ if (TransA == CblasTrans) {
+ sgemv_loop(i, j, N, M);
+ } else {
+ sgemv_loop(j, i, M, N);
+ }
+}
+
+static void scopy_raw(const unsigned int N, const float *X, const int incX,
+ float *Y, const int incY) {
+ unsigned int incy = abs(incY);
+ unsigned int incx = abs(incX);
+
+ for (unsigned int i = 0; i < N; ++i)
+ Y[i * incy] = X[i * incx];
+}
+
+static void sscal_raw(const unsigned int N, const float alpha, float *X,
+ const int incX) {
+ unsigned int incx = abs(incX);
+
+ for (unsigned int i = 0; i < N; ++i)
+ X[i * incx] = alpha * X[i * incx];
+}
+
+static float snrm2_raw(const unsigned int N, const float *X, const int incX) {
+ unsigned int incx = abs(incX);
+ float sum = 0.0f;
+ float tmp;
+#pragma omp parallel for private(tmp) reduction(+ : sum)
+ for (unsigned int i = 0; i < N; i++) {
+ tmp = X[i * incx];
+ sum += tmp * tmp;
+ }
+ return sqrt(sum);
+}
+
+static void sgemm_raw(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA,
+ CBLAS_TRANSPOSE TransB, const unsigned int M,
+ const unsigned int N, const unsigned int K,
+ const float alpha, const float *A, const unsigned int lda,
+ const float *B, const unsigned int ldb, const float beta,
+ float *C, const unsigned int ldc) {
+
+ for (unsigned int m = 0; m < M; ++m) {
+ for (unsigned int n = 0; n < N; ++n) {
+ double c = 0.0;
+ float c_old = C[m * ldc + n];
+ for (unsigned int k = 0; k < K; ++k) {
+ float a, b;
+ a = ((TransA == CblasTrans) ? A[k * lda + m] : A[m * lda + k]);
+ b = ((TransB == CblasTrans) ? B[n * ldb + k] : B[k * ldb + n]);
+ c += a * b;
+ }
+ C[m * ldc + n] = alpha * c;
+ if (beta != 0.0)
+ C[m * ldc + n] += beta * c_old;
+ }
+ }
+}
+
+#endif
+
+void saxpy(const unsigned int N, const float alpha, const float *X,
+ const int incX, float *Y, const int incY) {
+#ifdef USE_BLAS
+ cblas_saxpy(N, alpha, X, incX, Y, incY);
+#else
+ saxpy_raw(N, alpha, X, incX, Y, incY);
+#endif
+}
+
+void sgemm(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, CBLAS_TRANSPOSE TransB,
+ const unsigned int M, const unsigned int N, const unsigned int K,
+ const float alpha, const float *A, const unsigned int lda,
+ const float *B, const unsigned int ldb, const float beta, float *C,
+ const unsigned int ldc) {
+
+#ifdef USE_CUBLAS
+ int devID = 0;
+ cudaDeviceProp deviceProp;
+ cudaGetDeviceProperties(&deviceProp, devID);
+ float *d_A, *d_B, *d_C;
+
+ unsigned int size_A = M * K * sizeof(float);
+ unsigned int size_B = K * N * sizeof(float);
+ unsigned int size_C = M * N * sizeof(float);
+
+ cudaMalloc((void **)&d_A, size_A);
+ cudaMalloc((void **)&d_B, size_B);
+ cudaMemcpy(d_A, A, size_A, cudaMemcpyHostToDevice);
+ cudaMemcpy(d_B, B, size_B, cudaMemcpyHostToDevice);
+ cudaMalloc((void **)&d_C, size_C);
+
+ cublasHandle_t handle;
+ cublasCreate(&handle);
+
+ cublasOperation_t transA = (TransA == CblasTrans) ? CUBLAS_OP_T : CUBLAS_OP_N;
+ cublasOperation_t transB = (TransB == CblasTrans) ? CUBLAS_OP_T : CUBLAS_OP_N;
+ cublasSgemm(handle, transA, transB, N, M, K, &alpha, d_B, N, d_A, K, &beta,
+ d_C, N);
+
+ cudaMemcpy(C, d_C, size_C, cudaMemcpyDeviceToHost);
+ cublasDestroy(handle);
+#elif defined USE_BLAS
+ cblas_sgemm(order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C,
+ ldc);
+#else
+ sgemm_raw(order, TransA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C,
+ ldc);
+#endif
+}
+
+void scopy(const unsigned int N, const float *X, const int incX, float *Y,
+ const int incY) {
+#ifdef USE_BLAS
+ cblas_scopy(N, X, incX, Y, incY);
+#else
+ scopy_raw(N, X, incX, Y, incY);
+#endif
+}
+
+void sscal(const int N, const float alpha, float *X, const int incX) {
+#ifdef USE_BLAS
+ cblas_sscal(N, alpha, X, incX);
+#else
+ sscal_raw(N, alpha, X, incX);
+#endif
+}
+
+float snrm2(const int N, const float *X, const int incX) {
+#ifdef USE_BLAS
+ return cblas_snrm2(N, X, incX);
+#else
+ return snrm2_raw(N, X, incX);
+#endif
+}
+
+void sgemv(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, const unsigned int M,
+ const unsigned int N, const float alpha, const float *A,
+ const unsigned int lda, const float *X, const int incX,
+ const float beta, float *Y, const int incY) {
+#ifdef USE_BLAS
+ return cblas_sgemv(order, TransA, M, N, alpha, A, lda, X, incX, beta, Y,
+ incY);
+#else
+ return sgemv_raw(order, TransA, M, N, alpha, A, lda, X, incX, beta, Y, incY);
+#endif
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ *
+ * @file blas_interface.h
+ * @date 28 Aug 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is dummy header for blas support
+ *
+ */
+
+#ifndef __BLAS_INTERFACE_H_
+#define __BLAS_INTERFACE_H_
+#ifdef __cplusplus
+
+#ifdef USE_BLAS
+extern "C" {
+#include <cblas.h>
+}
+#else
+enum CBLAS_ORDER { CblasRowMajor = 101, CblasColMajor = 102 };
+
+enum CBLAS_TRANSPOSE {
+ CblasNoTrans = 111,
+ CblasTrans = 112,
+ CblasConjTrans = 113
+};
+#endif
+
+#ifdef USE_CUBLAS
+#include <helper_cuda.h>
+#include <helper_functions.h>
+#endif
+
+namespace nntrainer {
+
+/* TODO : need to scopy, sscal, snrm2 */
+void sscal(const int N, const float alpha, float *X, const int incX);
+
+float snrm2(const int N, const float *X, const int incX);
+
+void scopy(const unsigned int N, const float *X, const int incX, float *Y,
+ const int intY);
+
+void saxpy(const unsigned int N, const float alpha, const float *X,
+ const int incX, float *Y, const int incY);
+
+void sgemm(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, CBLAS_TRANSPOSE TransB,
+ const unsigned int M, const unsigned int N, const unsigned int K,
+ const float alpha, const float *A, const unsigned int lda,
+ const float *B, const unsigned int ldb, const float beta, float *C,
+ const unsigned int ldc);
+
+void sgemv(CBLAS_ORDER order, CBLAS_TRANSPOSE TransA, const unsigned int M,
+ const unsigned int N, const float alpha, const float *A,
+ const unsigned int lda, const float *X, const int incX,
+ const float beta, float *Y, const int incY);
+
+} /* namespace nntrainer */
+#endif /* __cplusplus */
+#endif /* __BLAS_INTERFACE_H__ */
--- /dev/null
+/* SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright (C) 2020 Jihoon Lee <jihoon.it.lee@samsung.com>
+ *
+ * @file lazy_tensor.cpp
+ * @date 05 Jun 2020
+ * @brief A lazy evaluation calculator for tensors
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jihoon Lee <jihoon.it.lee@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Wrapper method of add_i (immediate version of add)
+ * @retval this
+ */
+LazyTensor &LazyTensor::add_i(float const &value) {
+ call_chain.push_back(
+ [value](Tensor &t) mutable -> int { return t.add_i(value); });
+ return *this;
+}
+/**
+ * @brief Wrapper method of add_i. see tensor.h for more detail
+ * @param[in] m Tensor to be added
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::add_i(Tensor const &m, float const alpha) {
+ auto f = [&m, alpha](Tensor &t) mutable -> int { return t.add_i(m, alpha); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of subtract_i. see tensor.h for more detail
+ * @param[in] m Tensor to subtract
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::subtract_i(Tensor const &m) {
+ auto f = [&m](Tensor &t) mutable -> int { return t.subtract_i(m); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of subtract_i. see tensor.h for more detail
+ * @param[in] value value to subtract
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::subtract_i(float const &value) {
+ auto f = [value](Tensor &t) mutable -> int { return t.subtract_i(value); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of multiply_i. see tensor.h for more detail
+ * @param[in] value to be added
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::multiply_i(float const &value) {
+ auto f = [value](Tensor &t) mutable -> int { return t.multiply_i(value); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of multiply_i. see tensor.h for more detail
+ * @param[in] m Tensor to be multiplied
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::multiply_i(Tensor const &m) {
+ auto f = [&m](Tensor &t) mutable -> int { return t.multiply_i(m); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of divide_i. see tensor.h for more detail
+ * @param[in] value divisor
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::divide_i(float const &value) {
+ auto f = [value](Tensor &t) mutable -> int { return t.divide_i(value); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of divide_i. see tensor.h for more detail
+ * @param[in] m Tensor to for division
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::divide_i(Tensor const &m) {
+ auto f = [&m](Tensor &t) mutable -> int { return t.divide_i(m); };
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of dot. see tensor.h for more detail (memcopy
+ * happens)
+ * @param[in] m Tensor
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::dot(Tensor const &m) {
+ auto f = [&m](Tensor &t) mutable -> int {
+ try {
+ t = t.dot(m);
+ return ML_ERROR_NONE;
+ } catch (std::runtime_error &e) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ };
+
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of transpose. see tensor.h for more detail (memcopy
+ * happens)
+ * @param[in] direction to transpose ex) 0:2:1
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::transpose(std::string direction) {
+ auto f = [direction](Tensor &t) mutable -> int {
+ try {
+ t = t.transpose(direction);
+ return ML_ERROR_NONE;
+ } catch (std::runtime_error &e) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ };
+
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of sum. see tensor.h for more detail (memcopy
+ * happens)
+ * @param[in] direction to transpose ex) 0:2:1
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::sum_by_batch() {
+ auto f = [](Tensor &t) mutable -> int {
+ try {
+ t = t.sum_by_batch();
+ return ML_ERROR_NONE;
+ } catch (std::runtime_error &e) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ };
+
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of sum. see tensor.h for more detail (memcopy
+ * happens) 0 : batch direction 1 : channel direction 2 : channel direction 3 :
+ * channel direction
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::sum(int axis) {
+ auto f = [axis](Tensor &t) mutable -> int {
+ try {
+ t = t.sum(axis);
+ return ML_ERROR_NONE;
+ } catch (std::runtime_error &e) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ };
+
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of average. see tensor.h for more detail (memcopy
+ * happens)
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::average(int axis) {
+ auto f = [axis](Tensor &t) mutable -> int {
+ try {
+ t = t.average(axis);
+ return ML_ERROR_NONE;
+ } catch (std::runtime_error &e) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ };
+
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief Wrapper method of average. see tensor.h for more detail (memcopy
+ * happens)
+ * @retval LazyTensor *this
+ */
+LazyTensor &LazyTensor::average() {
+ auto f = [](Tensor &t) mutable -> int {
+ try {
+ t = t.average();
+ return ML_ERROR_NONE;
+ } catch (std::runtime_error &e) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ };
+
+ call_chain.push_back(f);
+ return *this;
+}
+
+/**
+ * @brief execute the call_chain to evaluate
+ * @retval calculated tensor
+ */
+Tensor LazyTensor::run() {
+ int status;
+ for (auto item : call_chain) {
+ status = item(target);
+ if (status != ML_ERROR_NONE) {
+ throw std::runtime_error("Error: evaluation failed");
+ }
+ }
+ return target;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/* Copyright (C) 2020 Jihoon Lee <jihoon.it.lee@samsung.com>
+ *
+ * @file lazy_tensor.h
+ * @date 05 Jun 2020
+ * @brief A lazy evaluation calculator for tensors
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jihoon Lee <jihoon.it.lee@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __LAZY_TENSOR_H__
+#define __LAZY_TENSOR_H__
+#ifdef __cplusplus
+
+#include <tensor.h>
+#include <vector>
+
+namespace nntrainer {
+
+/**
+ * @class LazyTensor a wrapper class for lazy calculation of tensor
+ * @brief calculation is delayed until Tensor LazyTensor::run() is
+ * called, can be contructed by Tensor::chain() method
+ */
+class LazyTensor {
+public:
+ /**
+ * @brief Constructor of Lazy Tensor, Tensor is copied to gaurantee
+ * immutability
+ */
+ LazyTensor(const Tensor &from) { target.copy(from); };
+
+ /**
+ * @brief Wrapper method of add_i. see tensor.h for more detail
+ * @param[in] value to be added
+ * @retval LazyTensor *this
+ */
+ LazyTensor &add_i(float const &value);
+
+ /**
+ * @brief Wrapper method of add_i. see tensor.h for more detail
+ * @param[in] m Tensor to be added
+ * @retval LazyTensor *this
+ */
+ LazyTensor &add_i(Tensor const &m, float const alpha = 1);
+
+ /**
+ * @brief Wrapper method of subtract_i. see tensor.h for more detail
+ * @param[in] m Tensor to subtract
+ * @retval LazyTensor *this
+ */
+ LazyTensor &subtract_i(Tensor const &m);
+
+ /**
+ * @brief Wrapper method of subtract_i. see tensor.h for more detail
+ * @param[in] value value to subtract
+ * @retval LazyTensor *this
+ */
+ LazyTensor &subtract_i(float const &value);
+
+ /**
+ * @brief Wrapper method of multiply_i. see tensor.h for more detail
+ * @param[in] value to be added
+ * @retval LazyTensor *this
+ */
+ LazyTensor &multiply_i(float const &value);
+
+ /**
+ * @brief Wrapper method of multiply_i. see tensor.h for more detail
+ * @param[in] m Tensor to be multiplied
+ * @retval LazyTensor *this
+ */
+ LazyTensor &multiply_i(Tensor const &m);
+
+ /**
+ * @brief Wrapper method of divide_i. see tensor.h for more detail
+ * @param[in] value divisor
+ * @retval LazyTensor *this
+ */
+ LazyTensor ÷_i(float const &value);
+
+ /**
+ * @brief Wrapper method of divide_i. see tensor.h for more detail
+ * @param[in] m Tensor to for division
+ * @retval LazyTensor *this
+ */
+ LazyTensor ÷_i(Tensor const &m);
+
+ /**
+ * @brief Wrapper method of dot. see tensor.h for more detail (memcopy
+ * happens)
+ * @param[in] m Tensor
+ * @retval LazyTensor *this
+ */
+ LazyTensor &dot(Tensor const &m);
+
+ /**
+ * @brief Wrapper method of transpose. see tensor.h for more detail
+ * (memcopy happens)
+ * @param[in] direction to transpose ex) 0:2:1
+ * @retval LazyTensor *this
+ */
+ LazyTensor &transpose(std::string direction);
+
+ /**
+ * @brief Wrapper method of sum_by_batch. see tensor.h for more detail
+ * (memcopy happens)
+ * @retval LazyTensor *this
+ */
+ LazyTensor &sum_by_batch();
+
+ /**
+ * @brief Wrapper method of sum. see tensor.h for more detail (memcopy
+ * happens) 0 : batch direction 1 : channel direction 2 : height direction 3 :
+ * width direction
+ * @retval LazyTensor *this
+ */
+ LazyTensor &sum(int axis);
+
+ /**
+ * @brief Wrapper method of average. see tensor.h for more detail (memcopy
+ * happens) 0 : batch direction 1 : channel direction 2 : height direction 3 :
+ * width direction
+ * @retval LazyTensor *this
+ */
+ LazyTensor &average(int axis);
+
+ /**
+ * @brief Wrapper method of average. see tensor.h for more detail (memcopy
+ * happens)
+ * @retval LazyTensor *this
+ */
+ LazyTensor &average();
+
+ /**
+ * @brief execute the call_chain to get the tensor
+ * @retval calculated tensor
+ */
+ Tensor run();
+
+private:
+ /**< handle the data as a std::vector type */
+ std::vector<std::function<int(Tensor &)>> call_chain;
+ Tensor target;
+};
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __LAZY_TENSOR_H__ */
--- /dev/null
+tensor_sources = [
+ 'blas_interface.cpp',
+ 'lazy_tensor.cpp',
+ 'tensor.cpp',
+ 'tensor_dim.cpp',
+ 'weight.cpp'
+]
+
+tensor_headers = [
+ 'tensor.h',
+ 'tensor_dim.h',
+ 'weight.h'
+]
+
+foreach s : tensor_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
+
+foreach h : tensor_headers
+ nntrainer_headers += join_paths(meson.current_source_dir(), h)
+endforeach
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file tensor.cpp
+ * @date 04 December 2019
+ * @brief This is Tensor class for calculation
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <assert.h>
+#include <cmath>
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <iterator>
+#include <random>
+#include <regex>
+#include <sstream>
+#include <stdio.h>
+
+#include <blas_interface.h>
+#include <lazy_tensor.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <parse_util.h>
+#include <tensor.h>
+#include <util_func.h>
+
+#define transposeloop(cl, ci, cj, ck, sl, si, sj, sk) \
+ do { \
+ unsigned int i, j, k, l; \
+ int inidx = 0, outidx = 0; \
+ for (cl = 0; cl < sl; cl++) \
+ for (ci = 0; ci < si; ci++) \
+ for (cj = 0; cj < sj; cj++) \
+ for (ck = 0; ck < sk; ck++) { \
+ outidx = si * sj * sk * cl + sj * sk * ci + sk * cj + ck; \
+ inidx = l * SI * SJ * SK + i * SJ * SK + j * SK + k; \
+ outptr[outidx] = inptr[inidx]; \
+ } \
+ } while (0);
+
+/** do clone of this, perform the operation and return the output */
+#define CLONE_OP_I(op, ...) \
+ do { \
+ Tensor clone = this->clone(); \
+ if (clone.op(__VA_ARGS__) != ML_ERROR_NONE) { \
+ std::stringstream ss; \
+ ss << "Error: op " << __func__ << " failed"; \
+ throw std::runtime_error(ss.str()); \
+ } \
+ return clone; \
+ } while (0);
+
+namespace nntrainer {
+
+static auto rng = [] {
+ std::mt19937 rng;
+ rng.seed(getSeed());
+ return rng;
+}();
+
+Tensor::Tensor(const TensorDim &d, const float *buf) : Tensor() {
+ if (d.getDataLen() != 0) {
+ dim = d;
+ strides = d.computeStrides();
+ data = std::shared_ptr<float>(new float[d.getDataLen()],
+ std::default_delete<float[]>());
+
+ if (buf != nullptr) {
+ float *data = getData();
+ unsigned int len = length();
+
+ scopy(len, buf, 1, data, 1);
+ }
+ }
+}
+
+bool Tensor::operator==(const Tensor &rhs) const {
+ if (this->dim != rhs.dim)
+ return false;
+
+ size_t len = length();
+
+ if (len != rhs.length())
+ return false;
+
+ const float *data = getData();
+ const float *rdata = rhs.getData();
+
+ for (size_t i = 0; i < len; ++i) {
+ if (std::isnan(data[i]) || std::isnan(rdata[i]) ||
+ std::fabs(data[i] - rdata[i]) > epsilon)
+ return false;
+ }
+
+ return true;
+}
+
+float Tensor::getValue(unsigned int batch, unsigned int c, unsigned int h,
+ unsigned int w) const {
+ return getData()[getIndex(batch, c, h, w)];
+}
+
+void Tensor::setValue(unsigned int batch, unsigned int c, unsigned int h,
+ unsigned int w, float value) {
+ if (!is_contiguous) {
+ throw std::runtime_error("cannot set value of non-contiguous tensor");
+ }
+
+ getData()[getIndex(batch, c, h, w)] = value;
+}
+
+template <typename T> void Tensor::setDist(T dist) {
+ float *data = getData();
+ unsigned int len = length();
+ for (unsigned int i = 0; i < len; ++i) {
+ data[i] = dist(rng);
+ }
+}
+
+void Tensor::setRandNormal(float mean, float std) {
+ setDist<std::normal_distribution<float>>(
+ std::normal_distribution<float>(mean, std));
+}
+
+void Tensor::setRandUniform(float min, float max) {
+ setDist<std::uniform_real_distribution<float>>(
+ std::uniform_real_distribution<float>(min, max));
+}
+
+Tensor::Tensor(
+ std::vector<std::vector<std::vector<std::vector<float>>>> const &d) {
+
+ if (d.empty() || d[0].empty() || d[0][0].empty() || d[0][0][0].empty()) {
+ throw std::out_of_range(
+ "[Tensor] trying to initialize Tensor from empty vector");
+ }
+
+ dim.batch(d.size());
+ dim.channel(d[0].size());
+ dim.height(d[0][0].size());
+ dim.width(d[0][0][0].size());
+ strides = dim.computeStrides();
+ data = std::shared_ptr<float>(new float[dim.getDataLen()],
+ std::default_delete<float[]>());
+ is_contiguous = true;
+
+ for (unsigned int i = 0; i < dim.batch(); ++i)
+ for (unsigned int j = 0; j < dim.channel(); ++j)
+ for (unsigned int k = 0; k < dim.height(); ++k)
+ for (unsigned int l = 0; l < dim.width(); ++l)
+ this->setValue(i, j, k, l, d[i][j][k][l]);
+}
+
+int Tensor::multiply_i(float const &value) {
+
+ float *data = getData();
+ unsigned int len = length();
+
+ sscal(len, value, data, 1);
+ return ML_ERROR_NONE;
+}
+
+Tensor Tensor::multiply(float const &value) { CLONE_OP_I(multiply_i, value); }
+
+int Tensor::divide_i(float const &value) {
+ if (value == 0.0f) {
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return this->multiply_i(1.0f / value);
+}
+
+Tensor Tensor::divide(float const &value) {
+ if (value == 0.0f) {
+ throw std::runtime_error("Error: Divide by zero");
+ }
+
+ CLONE_OP_I(divide_i, value);
+}
+
+int Tensor::add_i(float const &value) {
+ float *data = getData();
+ unsigned int len = length();
+
+ Tensor tmp(dim);
+ tmp.setValue(value);
+ saxpy(len, 1, tmp.getData(), 1, data, 1);
+
+ return ML_ERROR_NONE;
+}
+
+Tensor Tensor::add(float const &value) { CLONE_OP_I(add_i, value); }
+
+/**
+ * @struct External Loop Info for broadcasted info
+ * @brief External Loop Info for broadcasted iteration. Please refer to
+ * DISABLED_private_external_loop_n in unittest_nntrainer_tensor.
+ * @note This should better be implemented in iterator fashion before used
+ * extensively.
+ */
+struct Tensor::BroadcastInfo {
+
+ /**
+ * @brief Construct a new External Loop Info object
+ *
+ */
+ BroadcastInfo() : strides{0, 0, 0, 0} {}
+
+ unsigned int buffer_size; /**< virtual size of the buffer */
+ int buffer_axis; /**< the smallest axis that should be looped.
+ -1 means no loop needed*/
+ std::array<unsigned int, MAXDIM>
+ strides; /**< modified strides for the loop */
+};
+
+/**
+ * @brief Add Tensor Element by Element without mem copy
+ * @param[in] m Tensor to be added
+ * #retval #ML_ERROR_NONE Successful
+ * #retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ * TODO: add axis rather doing add over the last two dimensions always
+ */
+int Tensor::add_i(Tensor const &m, float const alpha) {
+ auto f = [&](const BroadcastInfo &e, float *buf, const float *m_buf) {
+ saxpy(e.buffer_size, alpha, m_buf, e.strides[3], buf, strides[3]);
+ };
+
+ return operator_i(m, f);
+}
+
+Tensor Tensor::add(Tensor const &m, float const alpha) const {
+ CLONE_OP_I(add_i, m, alpha);
+}
+
+int Tensor::subtract_i(Tensor const &m) { return add_i(m, -1); }
+
+Tensor Tensor::subtract(Tensor const &m) const { return add(m, -1); }
+
+int Tensor::subtract_i(float const &value) { return this->add_i(-value); }
+
+Tensor Tensor::subtract(float const &value) { return this->add(-value); }
+
+Tensor Tensor::pow(float exponent) const {
+ return apply([=](float in) { return powf(in, exponent); });
+}
+
+int Tensor::operator_i(
+ Tensor const &m,
+ std::function<void(const BroadcastInfo &e, float *, const float *)> v_func) {
+
+ BroadcastInfo e;
+
+ /// shortcut to cover when dimension matches
+ /// note that buffer_size, the last stride is only used in v_func but it might
+ /// be changed
+ if (dim == m.dim) {
+ e.buffer_size = length();
+ e.strides[3] = 1;
+ v_func(e, getData(), m.getData());
+ return ML_ERROR_NONE;
+ }
+
+ try {
+ e = this->computeBroadcastInfo(m);
+ } catch (std::exception &err) {
+ ml_loge("%s %s", typeid(err).name(), err.what());
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return operator_i_util(m, v_func, e);
+}
+
+int Tensor::operator_i_util(
+ Tensor const &m,
+ std::function<void(const BroadcastInfo &e, float *, const float *)> v_func,
+ const BroadcastInfo &e, int cur_axis, unsigned int offset,
+ unsigned int m_offset) {
+
+ float *buf = this->getData();
+ const float *m_buf = m.getData();
+ int status = ML_ERROR_NONE;
+
+ if (e.buffer_axis == cur_axis) {
+ v_func(e, buf + offset, m_buf + m_offset);
+ return ML_ERROR_NONE;
+ }
+
+ cur_axis++;
+ for (unsigned int i = 0; i < dim.getTensorDim(cur_axis); ++i) {
+ unsigned int next_offset = offset + i * strides[cur_axis];
+ unsigned int next_m_offset = m_offset + i * e.strides[cur_axis];
+ status =
+ operator_i_util(m, v_func, e, cur_axis, next_offset, next_m_offset);
+ if (status != ML_ERROR_NONE) {
+ ml_loge("[operator_i] failed: %d", status);
+ return status;
+ }
+ }
+
+ return status;
+}
+
+int Tensor::multiply_i(Tensor const &m) {
+ auto f = [&](const BroadcastInfo &e, float *buf, const float *m_buf) {
+ for (unsigned int i = 0; i < e.buffer_size; ++i) {
+ *buf *= *m_buf;
+ buf += strides[3];
+ m_buf += e.strides[3];
+ }
+ };
+
+ return operator_i(m, f);
+}
+
+Tensor Tensor::multiply(Tensor const &m) const { CLONE_OP_I(multiply_i, m); }
+
+int Tensor::divide_i(Tensor const &m) {
+ auto f = [&](const BroadcastInfo &e, float *buf, const float *m_buf) {
+ for (unsigned int i = 0; i < e.buffer_size; ++i) {
+ *buf /= *m_buf;
+ buf += strides[3];
+ m_buf += e.strides[3];
+ }
+ };
+
+ return operator_i(m, f);
+}
+
+Tensor Tensor::divide(Tensor const &m) const { CLONE_OP_I(divide_i, m); }
+
+/**
+ * This is to sum the Tensor data according to the dim.batch().
+ * Therefore the result has M(dim.batch(), 1, 1, 1) dimension.
+ */
+Tensor Tensor::sum_by_batch() {
+ Tensor ret(dim.batch(), 1, 1, 1);
+ unsigned int feat_len = dim.getFeatureLen();
+ unsigned int batch = dim.batch();
+
+ const float *data = getData();
+ float *rdata = ret.getData();
+
+ Tensor ones(1, 1, 1, feat_len);
+ ones.setValue(1.0);
+ sgemv(CblasRowMajor, CblasNoTrans, batch, feat_len, 1, data, feat_len,
+ ones.getData(), 1, 0.0, rdata, 1);
+
+ return ret;
+}
+
+/**
+ * @brief Calculate sum according to the axis.
+ */
+Tensor Tensor::sum(unsigned int axis, float alpha) const {
+ Tensor ret;
+
+ const float *data = getData();
+
+ if (axis >= 4)
+ throw std::out_of_range("Error: axis is invalid");
+
+ if (dim.getDim()[axis] == 1 and alpha == 1.0)
+ return this->clone();
+
+ switch (axis) {
+ case 0: {
+ ret = Tensor(1, dim.channel(), dim.height(), dim.width());
+ unsigned int feat_len = dim.getFeatureLen();
+ unsigned int batch = dim.batch();
+ Tensor ones(1, 1, 1, batch);
+ ones.setValue(alpha);
+ sgemv(CblasRowMajor, CblasTrans, batch, feat_len, 1, data, feat_len,
+ ones.getData(), 1, 0.0, ret.getData(), 1);
+ } break;
+ case 1: {
+ ret = Tensor(dim.batch(), 1, dim.height(), dim.width());
+ unsigned int feat_len = dim.height() * dim.width();
+ unsigned int channel = dim.channel();
+ Tensor ones(1, 1, 1, channel);
+ ones.setValue(alpha);
+ float *rdata = ret.getData();
+ for (unsigned int k = 0; k < dim.batch(); ++k) {
+ sgemv(CblasRowMajor, CblasTrans, channel, feat_len, 1,
+ &data[k * dim.getFeatureLen()], feat_len, ones.getData(), 1, 0.0,
+ &rdata[k * feat_len], 1);
+ }
+ } break;
+ case 2: {
+ ret = Tensor(dim.batch(), dim.channel(), 1, dim.width());
+ unsigned int width = dim.width();
+ unsigned int height = dim.height();
+ Tensor ones(1, 1, 1, height);
+ ones.setValue(alpha);
+ float *rdata = ret.getData();
+ for (unsigned int k = 0; k < dim.batch(); ++k) {
+ for (unsigned int c = 0; c < dim.channel(); ++c) {
+ unsigned int idx =
+ k * dim.getFeatureLen() + c * dim.width() * dim.height();
+ unsigned int ridx = k * ret.dim.getFeatureLen() + c * dim.width();
+ sgemv(CblasRowMajor, CblasTrans, height, width, 1, &data[idx], width,
+ ones.getData(), 1, 0.0, &rdata[ridx], 1);
+ }
+ }
+ } break;
+ case 3: {
+ ret = Tensor(dim.batch(), dim.channel(), dim.height(), 1);
+ unsigned int m = ret.dim.getDataLen();
+ unsigned int n = dim.width();
+ Tensor ones(1, 1, 1, n);
+ ones.setValue(alpha);
+ sgemv(CblasRowMajor, CblasNoTrans, m, n, 1, data, n, ones.getData(), 1, 0.0,
+ ret.getData(), 1);
+ } break;
+ default:
+ throw std::out_of_range("Error: Dimension cannot exceed 3");
+ }
+ return ret;
+}
+
+Tensor Tensor::sum(const std::vector<unsigned int> &axes, float alpha) const {
+ if (axes.empty())
+ throw std::invalid_argument("empty axes given");
+
+ Tensor ret = this->sum(axes[0], alpha);
+
+ for (unsigned int i = 1; i < axes.size(); ++i)
+ ret = ret.sum(axes[i]);
+
+ return ret;
+}
+
+/**
+ * @note: This dot product flattens the fist 3 axis for the purpose of
+ * computation. So, while performing, these matrices are behaving as 2-D
+ * matrices. The dimensions are restored while returning back the tensor.
+ */
+Tensor Tensor::dot(Tensor const &m, bool trans, bool trans_m) const {
+ if (m.dim.rank() > 2) {
+ throw exception::not_supported("Error: support only for rank of dot "
+ "matrix <= 2");
+ }
+ if (trans && dim.rank() > 2) {
+ throw exception::not_supported("Error: support only for rank of dot "
+ "matrix <= 2 with trans");
+ }
+
+ unsigned int dim1 = batch() * channel() * height();
+ unsigned int dim2 = width();
+ unsigned int mdim1 = m.batch() * m.channel() * m.height();
+ unsigned int mdim2 = m.width();
+ Tensor result;
+
+ unsigned int M, N, K, lda, ldb, ldc;
+
+ if (!trans && !trans_m) {
+ if (dim2 != mdim1)
+ throw std::runtime_error(
+ "Error: incompatible dimensions for dot product");
+ K = mdim1; /** == dim2 */
+ N = mdim2;
+ M = dim1;
+ lda = K;
+ ldb = N;
+ result = Tensor(batch(), channel(), height(), mdim2);
+
+ // We are not set zero the result because of performnace reason.
+ // However, result is not initialized properly. There might include garbage
+ // like nan. When we have to use this value as in C = alpha*A*B + beta*C,
+ // then have to check gargabe data of C is not effect or not.
+
+ } else if (!trans && trans_m) {
+ if (dim2 != mdim2)
+ throw std::runtime_error(
+ "Error: incompatible dimensions for dot product");
+ K = mdim2; /** == dim2 */
+ N = mdim1;
+ M = dim1;
+ lda = K;
+ ldb = K;
+ result = Tensor(batch(), channel(), height(), mdim1);
+ } else if (trans && !trans_m) {
+ if (dim1 != mdim1)
+ throw std::runtime_error(
+ "Error: incompatible dimensions for dot product");
+ K = mdim1; /** == dim1 */
+ N = mdim2;
+ M = dim2;
+ lda = M;
+ ldb = N;
+ result = Tensor(1, 1, dim2, mdim2);
+ } else {
+ if (dim1 != mdim2)
+ throw std::runtime_error(
+ "Error: incompatible dimensions for dot product");
+ K = mdim2; /** == dim1 */
+ N = mdim1;
+ M = dim2;
+ lda = M;
+ ldb = K;
+ result = Tensor(1, 1, dim2, mdim1);
+ }
+ ldc = N;
+
+ const float *data = getData();
+ const float *mdata = m.getData();
+ float *rdata = result.getData();
+ const float alpha = 1.0f;
+ const float beta = 0.0f;
+
+ enum CBLAS_TRANSPOSE transA = trans ? CblasTrans : CblasNoTrans;
+ enum CBLAS_TRANSPOSE transB = trans_m ? CblasTrans : CblasNoTrans;
+ sgemm(CblasRowMajor, transA, transB, M, N, K, alpha, data, lda, mdata, ldb,
+ beta, rdata, ldc);
+
+ return result;
+}
+
+Tensor Tensor::transpose(std::string direction) const {
+ unsigned int SL, SI, SJ, SK;
+ int dir[MAXDIM - 1];
+ unsigned int fromDim[4];
+ const float *inptr;
+ float *outptr;
+
+ fromDim[0] = dim.batch();
+ fromDim[1] = dim.channel();
+ fromDim[2] = dim.height();
+ fromDim[3] = dim.width();
+
+ getValues(3, direction, dir);
+ Tensor result(dim.batch(), fromDim[dir[0] + 1], fromDim[dir[1] + 1],
+ fromDim[dir[2] + 1]);
+
+ int indexI = dir[0];
+ int indexJ = dir[1];
+
+ SL = fromDim[0], SI = fromDim[1], SJ = fromDim[2], SK = fromDim[3];
+
+ inptr = getData();
+ outptr = result.getData();
+
+ switch (indexI) {
+ case 0:
+ if (indexJ == 1) {
+ transposeloop(l, i, j, k, SL, SI, SJ, SK);
+ } else {
+ transposeloop(l, i, k, j, SL, SI, SK, SJ);
+ }
+ break;
+ case 1:
+ if (indexJ == 0) {
+ transposeloop(l, j, i, k, SL, SJ, SI, SK);
+ } else {
+ transposeloop(l, j, k, i, SL, SJ, SK, SI);
+ }
+ break;
+ case 2:
+ if (indexJ == 0) {
+ transposeloop(l, k, i, j, SL, SK, SI, SJ);
+ } else {
+ transposeloop(l, k, j, i, SL, SK, SJ, SI);
+ }
+ break;
+ }
+
+ return result;
+}
+
+Tensor Tensor::apply(std::function<float(float)> f) const {
+ Tensor result(dim.batch(), dim.channel(), dim.height(), dim.width());
+ const float *data = getData();
+ float *rdata = result.getData();
+
+ std::transform(data, data + length(), rdata, f);
+
+ return result;
+}
+
+Tensor Tensor::apply(std::function<Tensor(Tensor)> f) const { return f(*this); }
+
+void Tensor::print(std::ostream &out) const {
+ printInstance(out, this);
+ const float *data = getData();
+
+ unsigned int len = length();
+
+ out << dim;
+
+ if (len > 100) {
+ out << '[' << data[0] << ' ' << data[1] << ' ' << data[2] << " ... "
+ << data[len - 3] << ' ' << data[len - 2] << ' ' << data[len - 1] << ']'
+ << std::endl;
+ return;
+ }
+
+ std::ios init(NULL);
+ init.copyfmt(out);
+ for (unsigned int k = 0; k < dim.batch(); k++) {
+ for (unsigned int l = 0; l < dim.channel(); l++) {
+ for (unsigned int i = 0; i < dim.height(); i++) {
+ for (unsigned int j = 0; j < dim.width(); j++) {
+ out << std::setw(10) << std::setprecision(10)
+ << this->getValue(k, l, i, j) << " ";
+ }
+ out << std::endl;
+ }
+ out << std::endl;
+ }
+ out << "-------" << std::endl;
+ }
+ out.copyfmt(init);
+}
+
+std::ostream &operator<<(std::ostream &out, Tensor const &m) {
+ m.print(out);
+ return out;
+}
+
+float *Tensor::getAddress(unsigned int i) {
+ if (i > this->dim.getDataLen()) {
+ ml_loge("Error: Index out of bounds");
+ return nullptr;
+ }
+
+ return &getData()[i];
+}
+
+const float *Tensor::getAddress(unsigned int i) const {
+ if (i > this->dim.getDataLen()) {
+ ml_loge("Error: Index out of bounds");
+ return nullptr;
+ }
+
+ return &getData()[i];
+}
+
+void Tensor::copy(const Tensor &from) {
+ // todo: enable copy to non-contiguous tensor
+ if (!is_contiguous) {
+ throw std::runtime_error("Cannot copy non-contiguous tensor");
+ }
+
+ Tensor t = Tensor(from.getDim(), from.getData());
+ swap(t, *this);
+}
+
+Tensor Tensor::clone() const {
+ Tensor t;
+ t.copy(*this);
+ return t;
+}
+
+void Tensor::reshape(const TensorDim &d) {
+ if (d.getDataLen() != dim.getDataLen()) {
+ throw std::invalid_argument("Error: reshape cannot change the tensor size");
+ }
+ dim = d;
+ strides = d.computeStrides();
+}
+
+void Tensor::save(std::ofstream &file) {
+ file.write((char *)getData(), getSize());
+}
+
+void Tensor::read(std::ifstream &file) {
+ file.read((char *)getData(), getSize());
+}
+
+/**
+ * @brief Calculate average value according to the axis.
+ */
+Tensor Tensor::average(unsigned int axis) const {
+ if (axis >= MAXDIM)
+ throw std::out_of_range(
+ "negative axis or axis more then MAXDIM is invalid");
+
+ unsigned int axis_size = dim.getDim()[axis];
+ if (axis_size == 1)
+ return this->clone();
+
+ return this->sum(axis, 1.0 / ((float)axis_size));
+}
+
+Tensor Tensor::average(const std::vector<unsigned int> &axes) const {
+ if (axes.empty())
+ return this->average();
+
+ TensorDim ret_shape;
+ for (const auto &idx : axes) {
+ if (idx >= MAXDIM) {
+ throw std::out_of_range("axis more then MAXDIM is invalid");
+ }
+ ret_shape.setTensorDim(idx, dim.getTensorDim(idx));
+ }
+
+ return this->sum(axes, 1.0 / (float)ret_shape.getDataLen());
+}
+
+/**
+ * @brief Calculate average value according to the axis.
+ */
+Tensor Tensor::average() const {
+ Tensor result = *this;
+ result.reshape({1, 1, 1, dim.getDataLen()});
+ return result.average(3);
+}
+
+void Tensor::setValue(float val) {
+ float *data = getData();
+ std::fill(data, data + length(), val);
+}
+
+void Tensor::setZero() { setValue(0); }
+
+std::vector<unsigned int> Tensor::argmax() const {
+ const float *data = getData();
+ std::vector<unsigned int> result;
+ unsigned int batch_size = batch();
+ unsigned int feature_len = dim.getFeatureLen();
+
+ result.reserve(batch_size);
+
+ for (unsigned int b = 0; b < batch_size; b++) {
+ auto max_iter =
+ std::max_element(data + b * feature_len, data + (b + 1) * feature_len);
+ result[b] = std::distance(data, max_iter);
+ }
+
+ return result;
+}
+
+float Tensor::l2norm() const {
+ unsigned int len = length();
+ const float *data = getData();
+
+ return snrm2(len, data, 1);
+}
+
+Tensor Tensor::normalization() const {
+ const float *data = getData();
+
+ auto bounds = std::minmax_element(data, data + length());
+ const float min = *bounds.first;
+ const float max = *bounds.second;
+
+ if (max == min) {
+ return this->subtract(*this);
+ }
+ return this->chain().subtract_i(min).divide_i(max - min).run();
+}
+
+LazyTensor Tensor::chain() const { return LazyTensor(*this); }
+
+Tensor Tensor::standardization() const {
+ Tensor result(dim);
+
+ const float *data = getData();
+ float *rdata = result.getData();
+
+ for (unsigned int k = 0; k < dim.batch(); ++k) {
+ int K = k * dim.getFeatureLen();
+ float mean;
+ float mean_tmp = 0.0f;
+ float std_tmp = 0.0f;
+ float std_dev = 0.0f;
+
+ for (unsigned int l = 0; l < dim.channel(); ++l) {
+ unsigned int L = K + l * dim.height() * dim.width();
+ for (unsigned int i = 0; i < dim.height(); ++i) {
+ unsigned int I = L + i * dim.width();
+ for (unsigned int j = 0; j < dim.width(); ++j) {
+ unsigned int J = I + j;
+ mean_tmp += data[J];
+ }
+ }
+ }
+
+ mean = mean_tmp / (this->dim.getFeatureLen());
+
+ for (unsigned int l = 0; l < dim.channel(); ++l) {
+ unsigned int L = K + l * dim.height() * dim.width();
+ for (unsigned int i = 0; i < dim.height(); ++i) {
+ unsigned int I = L + i * dim.width();
+ for (unsigned int j = 0; j < dim.width(); ++j) {
+ unsigned int J = I + j;
+ std_tmp += (data[J] - mean) * (data[J] - mean);
+ }
+ }
+ }
+ std_dev = sqrt(std_tmp) / (this->dim.getFeatureLen());
+
+ for (unsigned int l = 0; l < dim.channel(); ++l) {
+ unsigned int L = K + l * dim.height() * dim.width();
+ for (unsigned int i = 0; i < dim.height(); ++i) {
+ unsigned int I = L + i * dim.width();
+ for (unsigned int j = 0; j < dim.width(); ++j) {
+ unsigned int J = I + j;
+ rdata[J] = (data[J] - mean) / std_dev;
+ }
+ }
+ }
+ }
+
+ return result;
+}
+
+Tensor::BroadcastInfo Tensor::computeBroadcastInfo(const Tensor &m) {
+ if (m.length() > this->length())
+ throw exception::not_supported("broadcasting *this is not supported");
+
+ const TensorDim m_dim = m.getDim();
+
+ BroadcastInfo e;
+
+ /// checking if given Tensor's can be broadcasted
+ for (unsigned int i = 0; i < MAXDIM; ++i) {
+ if (dim.getTensorDim(i) == m_dim.getTensorDim(i)) {
+ e.strides[i] = m.strides[i];
+ continue;
+ }
+
+ /// If given dimension is 1, it could be reuesed, the stride remaining 0
+ /// Need to check if dim[i] == 1 && m_dim[i] == 1 first though
+ /// If so, strides should not change
+ if (m_dim.getTensorDim(i) == 1) {
+ continue;
+ }
+
+ std::stringstream ss;
+ ss << "[computeBroadcastInfo] broadcasting only allowed for"
+ "dimension value of 1 \n"
+ << "this: " << dim << "target: " << m_dim;
+ throw std::invalid_argument(ss.str().c_str());
+ }
+
+ /// calculate inner loop size
+ e.buffer_size = 1;
+ e.buffer_axis = -1;
+ e.strides[3] = m.strides[3];
+
+ /// initiate buffer info with matching dimension strategy
+ for (int axis = 3; axis >= 0; --axis) {
+ if (dim.getTensorDim(axis) != m_dim.getTensorDim(axis)) {
+ e.buffer_axis = axis;
+ break;
+ }
+
+ e.buffer_size *= dim.getTensorDim(axis);
+ }
+
+ /// check strategy that uses consecutive ones
+ if (m_dim.getTensorDim(3) == 1) {
+ unsigned int inner_loop_size = 1;
+ int axis;
+ for (axis = 3; axis >= 0; --axis) {
+ if (m_dim.getTensorDim(axis) != 1) {
+ break;
+ }
+
+ inner_loop_size *= dim.getTensorDim(axis);
+ }
+
+ /// if consecutive-one strategy has bigger chunk size, replace the
+ /// information
+ if (inner_loop_size > e.buffer_size) {
+ e.buffer_axis = axis;
+ e.buffer_size = inner_loop_size;
+ e.strides[3] = 0;
+ }
+ }
+
+ return e;
+}
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * @file tensor.h
+ * @date 04 December 2019
+ * @brief This is Tensor class for calculation
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __TENSOR_H__
+#define __TENSOR_H__
+#ifdef __cplusplus
+
+#include <array>
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include <tensor_dim.h>
+
+#define MAKE_SHARED_TENSOR(...) std::make_shared<nntrainer::Tensor>(__VA_ARGS__)
+
+namespace nntrainer {
+
+class LazyTensor;
+
+/**
+ * @class Tensor Class for Calculation
+ * @brief Tensor Class for Calculation
+ */
+class Tensor {
+public:
+ /**
+ * @brief Basic Constructor of Tensor
+ */
+ Tensor() :
+ dim(TensorDim()),
+ strides{dim.computeStrides()},
+ is_contiguous(true),
+ data(nullptr) {}
+
+ /**
+ * @brief Constructor of Tensor with dimension/buf
+ */
+ Tensor(const TensorDim &d, const float *buf = nullptr);
+
+ /**
+ * @brief Constructor of Tensor
+ * @param[in] batch Batch of Tensor
+ * @param[in] channel Channel of Tensor
+ * @param[in] height Height of Tensor
+ * @param[in] width Width of Tensor
+ */
+ Tensor(int batch, int channel, int height, int width) :
+ Tensor(TensorDim(batch, channel, height, width)){};
+
+ /**
+ * @brief Constructor of Tensor
+ * @param[in] channel Channel of Tensor
+ * @param[in] height Height of Tensor
+ * @param[in] width Width of Tensor
+ */
+ Tensor(int channel, int height, int width) :
+ Tensor(1, channel, height, width){};
+
+ /**
+ * @brief Constructor of Tensor with batch size one and channel size one
+ * @param[in] height Height of Tensor
+ * @param[in] width Width of Tensor
+ */
+ Tensor(int height, int width) : Tensor(1, 1, height, width){};
+
+ /**
+ * @brief Constructor of Tensor with just width
+ * @param[in] width Width of Tensor
+ */
+ Tensor(int width) : Tensor(1, 1, 1, width){};
+
+ /**
+ * @brief Constructor of Tensor
+ * @param[in] d data for the Tensor
+ */
+ Tensor(std::vector<std::vector<std::vector<std::vector<float>>>> const &d);
+
+ /**
+ * @brief Constructor of Tensor
+ * @note This constructor copies vector again. needs refactoring
+ * @param[in] d data for the Tensor
+ */
+ Tensor(std::vector<std::vector<std::vector<float>>> const &d) :
+ Tensor(std::vector<std::decay<decltype(d)>::type>{d}){};
+
+ /**
+ * @brief Constructor of Tensor
+ * @note This constructor copies vector again. needs refactoring
+ * @param[in] d data for the Tensor with batch size one
+ */
+ Tensor(std::vector<std::vector<float>> const &d) :
+ Tensor(std::vector<std::decay<decltype(d)>::type>{d}){};
+
+ /**
+ * @brief Copy constructor of Tensor.
+ * @param[in] Tensor &
+ */
+ Tensor(const Tensor &rhs) = default;
+
+ /**
+ * @brief Move constructor of Tensor.
+ * @param[in] Tensor &&
+ */
+ Tensor(Tensor &&rhs) noexcept = default;
+
+ /**
+ * @brief Copy assignment operator.
+ * @param[in] rhs Tensor to be copied.
+ */
+ Tensor &operator=(const Tensor &rhs) = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs Tensor to be moved.
+ */
+ Tensor &operator=(Tensor &&rhs) noexcept = default;
+
+ friend void swap(Tensor &lhs, Tensor &rhs) noexcept {
+ std::swap(lhs.dim, rhs.dim);
+ std::swap(lhs.data, rhs.data);
+ std::swap(lhs.strides, rhs.strides);
+ std::swap(lhs.is_contiguous, rhs.is_contiguous);
+ }
+
+ /**
+ * @brief Comparison operator overload
+ * @param[in] rhs Tensor to be compared with
+ */
+ bool operator==(const Tensor &rhs) const;
+
+ /**
+ * @brief Comparison operator overload
+ * @param[in] rhs Tensor to be compared with
+ */
+ bool operator!=(const Tensor &rhs) const { return !(*this == rhs); }
+
+ /**
+ * @brief return value at specific location
+ * @param[in] batch batch location
+ * @param[in] c channel location
+ * @param[in] h height location
+ * @param[in] w width location
+ */
+ float getValue(unsigned int batch, unsigned int c, unsigned int h,
+ unsigned int w) const;
+
+ /**
+ * @brief Multiply value element by element immediately
+ * @param[in] value multiplier
+ * @retval #ML_ERROR_INVALID_PARAMETER Tensor dimension is not right
+ * @retval #ML_ERROR_NONE Successful
+ */
+ int multiply_i(float const &value);
+
+ /**
+ * @brief Multiply value element by element
+ * @param[in] value multiplier
+ * @retval Calculated Tensor
+ */
+ Tensor multiply(float const &value);
+
+ /**
+ * @brief Divide value element by element immediately
+ * @param[in] value divisor
+ * @retval #ML_ERROR_INVALID_PARAMETER Tensor dimension is not right
+ * @retval #ML_ERROR_NONE Successful
+ */
+ int divide_i(float const &value);
+
+ /**
+ * @brief Divide value element by element
+ * @param[in] value Divisor
+ * @retval Calculated Tensor
+ */
+ Tensor divide(float const &value);
+
+ /**
+ * @brief Add Tensor Element by Element without mem copy
+ * @param[in] m Tensor to be added
+ * @param[out] alpha Values to be scaled
+ * @retval #ML_ERROR_NONE Successful
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ */
+ int add_i(Tensor const &m, float const alpha = 1);
+
+ /**
+ * @brief Add Tensor Element by Element
+ * @param[in] m Tensor to be added
+ * @retval Calculated Tensor
+ */
+ Tensor add(Tensor const &m, float const alpha = 1) const;
+
+ /**
+ * @brief Add Tensor Element immediately to target tensor without mem copy
+ * @param[in] value value to be added
+ * @retval #ML_ERROR_NONE Successful
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ */
+ int add_i(float const &value);
+
+ /**
+ * @brief Add value Element by Element
+ * @param[in] value value to be added
+ * @retval Calculated Tensor
+ */
+ Tensor add(float const &value);
+
+ /**
+ * @brief memcpyless version of subtract
+ * @param[in] m Tensor to be subtracted
+ * @retval #ML_ERROR_NONE Successful
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ */
+ int subtract_i(Tensor const &m);
+
+ /**
+ * @brief Substract Tensor Element by Element
+ * @param[in] m Tensor to be subtracted
+ * @retval Calculated Tensor
+ */
+ Tensor subtract(Tensor const &m) const;
+
+ /**
+ * @brief memcpyless version of subtract
+ * @param[in] value value to subtract
+ * @retval #ML_ERROR_NONE Successful
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ */
+ int subtract_i(float const &value);
+
+ /**
+ * @brief subtract value Element by Element
+ * @param[in] value value to be subtracted
+ * @retval Calculated Tensor
+ */
+ Tensor subtract(float const &value);
+
+ /**
+ * @brief Multiply Tensor Elementwise
+ * @param[in] m Tensor to be multiplied
+ * @retval #ML_ERROR_NONE successful
+ */
+ int multiply_i(Tensor const &m);
+
+ /**
+ * @brief Multiply Tensor Element by Element ( Not the MxM )
+ * @param[in] m Tensor to be multiplied
+ * @retval Calculated Tensor
+ */
+ Tensor multiply(Tensor const &m) const;
+
+ /**
+ * @brief divide Tensor Elementwise
+ * @param[in] m Tensor to be multiplied
+ * @retval #ML_ERROR_NONE successful
+ */
+ int divide_i(Tensor const &m);
+
+ /**
+ * @brief Divide Tensor Element by Element
+ * @param[in] m Divisor Tensor
+ * @retval Calculated Tensor
+ */
+ Tensor divide(Tensor const &m) const;
+
+ /**
+ * @brief Tensor power Element by Element
+ * @param[in] float Divisor Tensor
+ * @retval Calculated Tensor
+ */
+ Tensor pow(float m) const;
+
+ /**
+ * @brief Dot Product of Tensor ( equal MxM )
+ * @details This applies dot of the last dimension of this and second-last
+ * dimension of passed tensor m.
+ * @param[in] m Tensor
+ * @param[in] trans Transpose
+ * @param[in] trans_m Transpose m
+ * @retval Calculated Tensor
+ */
+ Tensor dot(Tensor const &m, bool trans = false, bool trans_m = false) const;
+
+ /**
+ * @brief Transpose Tensor
+ * @param[in] direction to transpose ex) 0:2:1
+ * @retval Calculated Tensor
+ */
+ Tensor transpose(std::string direction) const;
+
+ /**
+ * @brief sum all the Tensor elements according to the batch
+ * @retval Calculated Tensor(batch, 1, 1, 1)
+ */
+ Tensor sum_by_batch();
+
+ /**
+ * @brief sum all the Tensor elements according to the axis
+ * 0 : batch direction
+ * 1 : channel direction
+ * 2 : height direction
+ * 3 : width direction
+ * @param[in] axis Axis to calculate sum along
+ * @param[in] alpha Scale the sum by this value
+ * @retval Calculated Tensor
+ */
+ Tensor sum(unsigned int axis, float alpha = 1.0) const;
+
+ /**
+ * @brief sum all the Tensor by multiple axes
+ *
+ * @param axes axes to sum along
+ * @param alpha Scale the sum by this value
+ * @return Tensor
+ */
+ Tensor sum(const std::vector<unsigned int> &axes, float alpha = 1.0) const;
+
+ /**
+ * @brief Averaging the Tensor elements according to the axis
+ * 0 : batch direction
+ * 1 : channel direction
+ * 2 : height direction
+ * 3 : width direction
+ * @retval Calculated Tensor
+ */
+ Tensor average(unsigned int axis) const;
+
+ /**
+ * @brief average all the Tensor by multiple axes
+ *
+ * @param axes axes to sum along
+ * @return Tensor
+ */
+ Tensor average(const std::vector<unsigned int> &axes) const;
+
+ /**
+ * @brief Averaging the Tensor elements by all axis
+ * @retval Calculated Tensor
+ */
+ Tensor average() const;
+
+ /**
+ * @brief Anchor a starting point to defer following evaluation
+ * @retval LazyTensor class that can be used with run();
+ */
+ LazyTensor chain() const;
+
+ /**
+ * @brief Softmax the Tensor elements
+ * @retval Calculated Tensor
+ */
+ Tensor softmax() const;
+
+ /**
+ * @brief l2norm the Tensor elements
+ * @retval Calculated l2norm
+ */
+ float l2norm() const;
+
+ /**
+ * @brief Normalize the Tensor elements
+ * @retval Calculated Tensor
+ */
+ Tensor normalization() const;
+
+ /**
+ * @brief Standardize the Tensor elements
+ * @retval Calculated Tensor
+ */
+ Tensor standardization() const;
+
+ /**
+ * @brief Fill the Tensor elements with zero
+ */
+ void setZero();
+
+ /**
+ * @brief Apply function element by element
+ * @param[in] *function function pointer applied
+ * @retval Tensor
+ */
+ Tensor apply(std::function<float(float)> f) const;
+
+ /**
+ * @brief Apply function to Tensor
+ * @param[in] *function function pointer applied
+ * @retval Tensor
+ */
+ Tensor apply(std::function<Tensor(Tensor)> f) const;
+
+ Tensor apply_i(std::function<int(const Tensor &)> f) const;
+
+ /**
+ * @brief Print element
+ * @param[in] out out stream
+ * @retval Tensor
+ */
+ void print(std::ostream &out) const;
+
+ /**
+ * @brief Get length of current _data
+ * @retval unsigned int length of the current _data
+ */
+ unsigned int length() const { return dim.getDataLen(); }
+
+ /**
+ * @brief Get size of the data
+ * @retval size_t Size in bytes
+ */
+ size_t getSize() const { return length() * sizeof(float); }
+
+ /**
+ * @brief Set the element value
+ * @param[in] batch batch location
+ * @param[in] c channel location
+ * @param[in] i height location
+ * @param[in] j width location
+ * @param[in] value value to be stored
+ */
+ void setValue(unsigned int batch, unsigned int c, unsigned int i,
+ unsigned int j, float value);
+
+ /**
+ * @brief Fill the Tensor elements with value
+ * @param[in] value value to be stored
+ */
+ void setValue(float value);
+
+ /**
+ * @brief Set the tensor with random normal distribution
+ * @param[in] mean mean of the distribution
+ * @param[in] std standard deviation of the distribution
+ */
+ void setRandNormal(float mean = 0.0f, float std = 0.05f);
+
+ /**
+ * @brief Set the tensor with random uniform distribution
+ * @param[in] min minimum value for the distribution
+ * @param[in] max maximum value for the distribution
+ */
+ void setRandUniform(float min = -0.05f, float max = 0.05f);
+
+ /**
+ * @brief Copy the Tensor
+ * @param[in] from Tensor to be copied
+ */
+ void copy(const Tensor &from);
+
+ /**
+ * @brief Convient wrapper for inplace copy of @a this.
+ * @retval Copied version of this
+ */
+ Tensor clone() const;
+
+ /**
+ * @brief Save the Tensor into file
+ * @param[in] file output file stream
+ */
+ void save(std::ofstream &file);
+
+ /**
+ * @brief Read the Tensor from file
+ * @param[in] file input file stream
+ */
+ void read(std::ifstream &file);
+
+ /**
+ * @brief return argument index which value is max by batch
+ * @retval unsigned int argument index
+ */
+ std::vector<unsigned int> argmax() const;
+
+ /**
+ * @brief return a copy of the Tensor Dim
+ * @retval TensorDim
+ */
+ TensorDim getDim() const { return TensorDim(dim); }
+
+ /**
+ * @brief return Tensor Dim for a given axis
+ * @retval dimension
+ */
+ unsigned int getTensorDim(unsigned int axis);
+
+ /**
+ * @brief return Tensor batch size
+ * @retval batch size
+ */
+ unsigned int batch() const { return dim.batch(); }
+
+ /**
+ * @brief return Tensor batch size
+ * @retval batch size
+ */
+ unsigned int channel() const { return dim.channel(); }
+
+ /**
+ * @brief return Tensor height size
+ * @retval height size
+ */
+ unsigned int height() const { return dim.height(); }
+
+ /**
+ * @brief return Tensor batch size
+ * @retval width size
+ */
+ unsigned int width() const { return dim.width(); }
+
+ /**
+ * @brief return Data pointer of Tensor
+ * @retval float pointer
+ */
+ float *getData() { return data.get(); }
+
+ const float *getData() const { return data.get(); }
+
+ /**
+ * @brief i data index
+ * @retval address of ith data
+ */
+ float *getAddress(unsigned int i);
+
+ /**
+ * @brief i data index
+ * @retval address of ith data
+ */
+ const float *getAddress(unsigned int i) const;
+
+ /**
+ * @brief set Tensor Dim
+ * @param[in] d TensorDim
+ * @note Throws std::invalid_argument if size mismatch
+ */
+ void reshape(const TensorDim &d);
+
+ /**
+ * @brief return current stride of tensor.
+ * @retval int[MAXDIM] strides
+ */
+ const std::array<unsigned int, MAXDIM> getStrides() const noexcept {
+ return strides;
+ }
+
+ static constexpr float epsilon = 1e-5;
+
+private:
+ /**
+ * @brief Get linear index given the n-d index
+ */
+ inline unsigned int getIndex(unsigned int b, unsigned int c, unsigned int h,
+ unsigned int w) const {
+ return (b * strides[0] + c * strides[1] + h * strides[2] + w * strides[3]);
+ }
+
+ struct BroadcastInfo;
+
+ /**
+ * @brief Applies the given operator to the tensor with the passed argument
+ * @param[in] m Tensor
+ * @param[in] v_func vectorized function to apply
+ * @param e broadcast info.
+ * @param cur_axis current axis. pass default when calling outside.
+ * @param offset offset for this. pass default when calling outside.
+ * @param m_offset offset for m. pass default when calling outside.
+ * @retval #ML_ERROR_NONE Successful
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ */
+ int operator_i_util(
+ Tensor const &m,
+ std::function<void(const BroadcastInfo &e, float *, const float *)> v_func,
+ const BroadcastInfo &e, int cur_axis = -1, unsigned int offset = 0,
+ unsigned int m_offset = 0);
+
+ /**
+ * @brief Applies the given operator to the tensor with the passed argument
+ *
+ * @param[in] m Tensor
+ * @param[in] v_func vectorized function to apply
+ * @retval #ML_ERROR_NONE Successful
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter
+ */
+ int operator_i(
+ Tensor const &m,
+ std::function<void(const BroadcastInfo &e, float *, const float *)> v_func);
+
+ /**
+ * @brief compute Loop info for broadcasting and vectorization
+ *
+ * @param m target tensor to be calculated against.
+ * @return BroadcastInfo Loopinfo needed to run external loop
+ */
+ BroadcastInfo computeBroadcastInfo(const Tensor &m);
+
+ /**< handle the data as a std::shared_ptr<float> type */
+ TensorDim dim;
+ std::array<unsigned int, MAXDIM> strides;
+ bool is_contiguous;
+ std::shared_ptr<float> data;
+
+ template <typename T> void setDist(T dist);
+};
+
+/**
+ * @brief Overriding output stream
+ */
+std::ostream &operator<<(std::ostream &out, Tensor const &m);
+
+typedef std::shared_ptr<Tensor> sharedTensor;
+
+typedef std::shared_ptr<const Tensor> sharedConstTensor;
+
+typedef std::vector<sharedConstTensor> sharedConstTensors;
+
+typedef std::vector<sharedTensor> sharedTensors;
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __TENSOR_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ */
+/**
+ * @file tensor_dim.cpp
+ * @date 22 May 2020
+ * @brief This is Tensor Dimension Class
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <cstring>
+#include <regex>
+#include <sstream>
+#include <stdio.h>
+
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <tensor_dim.h>
+
+namespace nntrainer {
+
+TensorDim::TensorDim(const std::string &shape) : TensorDim() {
+ if (setTensorDim(shape) != ML_ERROR_NONE) {
+ throw std::invalid_argument("[TensorDim] Setting TensorDim failed");
+ }
+}
+
+TensorDim &TensorDim::operator=(const TensorDim &rhs) {
+ using std::swap;
+
+ TensorDim tmp(rhs.batch(), rhs.channel(), rhs.height(), rhs.width());
+ swap(*this, tmp);
+ return *this;
+}
+
+TensorDim &TensorDim::operator=(TensorDim &&rhs) noexcept {
+ using std::swap;
+
+ swap(*this, rhs);
+ return *this;
+}
+
+void TensorDim::resetLen() {
+ feature_len = dim[1] * dim[2] * dim[3];
+ len = dim[0] * feature_len;
+}
+
+const unsigned int TensorDim::getTensorDim(unsigned int idx) const {
+ if (idx >= MAXDIM)
+ throw std::invalid_argument(
+ "[TensorDim] Tensor Dimension index should be between 0 and 4");
+
+ return dim[idx];
+}
+
+void TensorDim::setTensorDim(unsigned int idx, unsigned int value) {
+ if (idx >= MAXDIM)
+ throw std::out_of_range(
+ "[TensorDim] Tensor Dimension index should be between 0 and 4");
+
+ if (value <= 0)
+ throw std::invalid_argument(
+ "[TensorDim] Trying to assign value <=0 to tensor dim");
+
+ if (len == 0) {
+ for (size_t i = 0; i < MAXDIM; ++i) {
+ dim[i] = 1;
+ }
+ }
+
+ dim[idx] = value;
+ resetLen();
+}
+
+int TensorDim::setTensorDim(const std::string &input_shape) {
+ int status = ML_ERROR_NONE;
+ std::regex words_regex("[^\\s.,:;!?]+");
+ auto words_begin =
+ std::sregex_iterator(input_shape.begin(), input_shape.end(), words_regex);
+ auto words_end = std::sregex_iterator();
+ int cur_dim = std::distance(words_begin, words_end);
+ if (cur_dim <= 0 || (size_t)cur_dim > MAXDIM) {
+ ml_loge("Tensor Dimension should be between 1 and 4");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ int cn = 0;
+ for (std::sregex_iterator i = words_begin; i != words_end; ++i, ++cn) {
+ setTensorDim(MAXDIM - cur_dim + cn, std::stoul((*i).str()));
+ }
+
+ return status;
+}
+
+bool TensorDim::operator==(const TensorDim &rhs) const {
+ for (size_t i = 0; i < MAXDIM; ++i) {
+ if (this->dim[i] != rhs.dim[i]) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+unsigned int TensorDim::rank() const {
+ unsigned int rank = 0;
+ for (unsigned int i = 0; i < MAXDIM; i++) {
+ if (dim[i] > 1)
+ rank += 1;
+ }
+ return rank;
+}
+
+unsigned int &TensorDim::operator[](const unsigned int index) {
+ if (index >= MAXDIM)
+ throw std::out_of_range(
+ "[TensorDim] Tensor Dimension index should be between 0 and 4");
+ return dim[index];
+}
+
+const unsigned int &TensorDim::operator[](const unsigned int index) const {
+ if (index >= MAXDIM)
+ throw std::out_of_range(
+ "[TensorDim] Tensor Dimension index should be between 0 and 4");
+ return dim[index];
+}
+
+void TensorDim::reverse() { std::reverse(dim, dim + MAXDIM); }
+
+std::ostream &operator<<(std::ostream &out, TensorDim const &d) {
+ out << "Shape: " << d.batch() << ":" << d.channel() << ":" << d.height()
+ << ":" << d.width() << std::endl;
+ return out;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
+ */
+/**
+ * @file tensor_dim.h
+ * @date 22 May 2020
+ * @brief This is Tensor Dimension Class
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __TENSOR_DIM_H__
+#define __TENSOR_DIM_H__
+#ifdef __cplusplus
+
+#include <array>
+#include <iostream>
+
+namespace nntrainer {
+
+constexpr const size_t MAXDIM = 4;
+
+class TensorDim {
+public:
+ TensorDim() {
+ for (size_t i = 0; i < MAXDIM; ++i) {
+ dim[i] = 0;
+ }
+ len = 0;
+ feature_len = 0;
+ }
+
+ TensorDim(unsigned int b, unsigned int c, unsigned int h, unsigned int w) :
+ TensorDim() {
+ setTensorDim(0, b);
+ setTensorDim(1, c);
+ setTensorDim(2, h);
+ setTensorDim(3, w);
+ feature_len = c * h * w;
+ len = b * feature_len;
+ }
+
+ TensorDim(const TensorDim &rhs) = default;
+
+ TensorDim(const std::string &shape);
+
+ ~TensorDim(){};
+
+ /**
+ * @brief Move constructor of Conv 2D Layer.
+ * @param[in] Conv2dLayer &&
+ */
+ TensorDim(TensorDim &&rhs) noexcept = default;
+
+ /**
+ * @brief Move assignment operator.
+ * @parma[in] rhs Optimizer to be moved.
+ */
+ TensorDim &operator=(TensorDim &&rhs) noexcept;
+
+ /**
+ * @brief swap variable of Conv2D Layer
+ * @parma[out] lhs Optimizer
+ * @parma[in] rhs Optimizer
+ */
+ friend void swap(TensorDim &lhs, TensorDim &rhs) noexcept {
+ std::swap_ranges(std::begin(lhs.dim), std::begin(lhs.dim) + MAXDIM,
+ std::begin(rhs.dim));
+ std::swap(lhs.len, rhs.len);
+ std::swap(lhs.feature_len, rhs.feature_len);
+ }
+
+ unsigned int batch() const { return dim[0]; };
+ unsigned int channel() const { return dim[1]; };
+ unsigned int height() const { return dim[2]; };
+ unsigned int width() const { return dim[3]; };
+ unsigned int getDataLen() const { return len; };
+ unsigned int getFeatureLen() const { return feature_len; };
+
+ void resetLen();
+ void batch(unsigned int b) { setTensorDim(0, b); }
+ void channel(unsigned int c) { setTensorDim(1, c); }
+ void height(unsigned int h) { setTensorDim(2, h); }
+ void width(unsigned int w) { setTensorDim(3, w); }
+
+ const unsigned int *getDim() const { return dim; }
+ unsigned int getNumDim() const { return MAXDIM; }
+
+ const unsigned int getTensorDim(unsigned int idx) const;
+ void setTensorDim(unsigned int idx, unsigned int value);
+ int setTensorDim(const std::string &input_shape);
+
+ TensorDim &operator=(const TensorDim &rhs);
+ bool operator==(const TensorDim &rhs) const;
+ bool operator!=(const TensorDim &rhs) const { return !(*this == rhs); }
+ bool isEmpty() const { return len == 0; }
+ unsigned int rank() const;
+
+ unsigned int &operator[](const unsigned int index);
+ const unsigned int &operator[](const unsigned int index) const;
+
+ /**
+ * @brief Calculate standard strides
+ *
+ * @return std::array <int, MAXDIM>
+ */
+ std::array<unsigned int, MAXDIM> computeStrides() const {
+ return {dim[1] * dim[2] * dim[3], dim[2] * dim[3], dim[3], 1};
+ }
+
+ /**
+ * @brief reverse the dimensions inplace
+ */
+ void reverse();
+
+private:
+ unsigned int dim[MAXDIM];
+ unsigned int len;
+ unsigned int feature_len;
+};
+
+std::ostream &operator<<(std::ostream &out, TensorDim const &d);
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __TENSOR_DIM_H__ */
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0-only
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file weight.cpp
+ * @date 22 September 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Weight Class for Neural Network
+ *
+ */
+
+#include <util_func.h>
+#include <weight.h>
+
+namespace nntrainer {
+
+Weight::Weight(const Weight &rhs) :
+ initializer(rhs.initializer),
+ trainable(rhs.trainable),
+ name(rhs.name) {
+ var = rhs.var.clone();
+ grad = rhs.grad.clone();
+}
+
+Weight &Weight::operator=(const Weight &rhs) {
+ Weight temp(rhs);
+ swap(temp, *this);
+ return *this;
+}
+
+Weight::Weight(const TensorDim &dim, const WeightInitializer init, bool train,
+ std::string name) :
+ initializer(init),
+ trainable(train),
+ name(name) {
+ if (initializer == WeightInitializer::WEIGHT_UNKNOWN)
+ throw std::invalid_argument("Weight initializer unknown");
+
+ initializeVar(dim);
+ if (trainable) {
+ grad = Tensor(dim);
+ grad.setZero();
+ } else
+ grad = Tensor();
+}
+
+void Weight::initializeVar(const TensorDim &dim) {
+ var = Tensor(dim);
+ switch (initializer) {
+ case WeightInitializer::WEIGHT_ZEROS:
+ var.setZero();
+ break;
+ case WeightInitializer::WEIGHT_ONES:
+ var.setValue(1.0f);
+ break;
+ case WeightInitializer::WEIGHT_LECUN_NORMAL:
+ var.setRandNormal(0.0f, sqrtFloat(1.0f / dim.height()));
+ break;
+ case WeightInitializer::WEIGHT_XAVIER_NORMAL:
+ var.setRandNormal(0.0f, sqrtFloat(2.0f / (dim.width() + dim.height())));
+ break;
+ case WeightInitializer::WEIGHT_HE_NORMAL:
+ var.setRandNormal(0.0f, sqrtFloat(2.0f / (dim.height())));
+ break;
+ case WeightInitializer::WEIGHT_LECUN_UNIFORM:
+ var.setRandUniform(-1.0f * sqrtFloat(1.0f / dim.height()),
+ sqrtFloat(1.0f / dim.height()));
+ break;
+ case WeightInitializer::WEIGHT_XAVIER_UNIFORM:
+ var.setRandUniform(-1.0f * sqrtFloat(6.0f / (dim.height() + dim.width())),
+ sqrtFloat(6.0 / (dim.height() + dim.width())));
+ break;
+ case WeightInitializer::WEIGHT_HE_UNIFORM:
+ var.setRandUniform(-1.0f * sqrtFloat(6.0f / (dim.height())),
+ sqrtFloat(6.0 / (dim.height())));
+ break;
+ default:
+ break;
+ }
+}
+
+} // namespace nntrainer
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0-only
+/**
+ * Copyright (C) 2020 Parichay Kapoor <pk.kapoor@samsung.com>
+ *
+ * @file weight.h
+ * @date 22 September 2020
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
+ * @bug No known bugs except for NYI items
+ * @brief This is Weight Class for Neural Network
+ *
+ */
+
+#ifndef __WEIGHT_H__
+#define __WEIGHT_H__
+
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @brief Enumeration of Weight Decay type
+ */
+enum class WeightRegularizerType {
+ l2norm, /** L2 norm regularizer */
+ unknown /** Unknown */
+};
+
+/**
+ * @brief Enumeration of Weight Initialization Type
+ */
+enum class WeightInitializer {
+ WEIGHT_ZEROS, /** Zero initialization */
+ WEIGHT_ONES, /** One initialization */
+ WEIGHT_LECUN_NORMAL, /** LeCun normal initialization */
+ WEIGHT_LECUN_UNIFORM, /** uniform initialization */
+ WEIGHT_XAVIER_NORMAL, /** Xavier normal initialization */
+ WEIGHT_XAVIER_UNIFORM, /** Xavier uniform initialization */
+ WEIGHT_HE_NORMAL, /** He normal initialization */
+ WEIGHT_HE_UNIFORM, /** He uniform initialization */
+ WEIGHT_UNKNOWN /** Unknown */
+};
+
+/**
+ * @class Weight
+ * @brief Weight with gradient, and its corresponding trainable property
+ */
+class Weight {
+
+ /** Declare layers as friend to get variable/gradient reference */
+ friend class Layer;
+ friend class Conv2DLayer;
+ friend class FullyConnectedLayer;
+ friend class BatchNormalizationLayer;
+
+ /** Declare opitmizer as friend to get variable/gradient reference */
+ friend class Optimizer;
+ friend class SGD;
+ friend class Adam;
+
+public:
+ /**
+ * @brief Weight default constructor
+ */
+ Weight() : initializer(WeightInitializer::WEIGHT_UNKNOWN), trainable(false) {}
+
+ /**
+ * @brief Construct a new Weight object
+ *
+ * @param dim Variable and gradient tensor dimension
+ * @param init Initializer for the tensor
+ * @param train If the variable is trainable
+ * @param name Name for this weight
+ */
+ Weight(
+ const TensorDim &dim,
+ const WeightInitializer init = WeightInitializer::WEIGHT_XAVIER_UNIFORM,
+ bool train = true, std::string name = "");
+
+ /**
+ * @brief Allocate and initialize the variable
+ *
+ * @param dim Dimension for the variable
+ */
+ void initializeVar(const TensorDim &dim);
+
+ /**
+ * @brief Swap for weight
+ *
+ * @param lhs Swap to
+ * @param rhs Swap from
+ * @note Only swap gradient if trainable
+ */
+ friend void swap(Weight &lhs, Weight &rhs) noexcept {
+ using std::swap;
+
+ swap(lhs.var, rhs.var);
+ swap(lhs.initializer, rhs.initializer);
+ swap(lhs.trainable, rhs.trainable);
+ swap(lhs.grad, rhs.grad);
+ swap(lhs.name, rhs.name);
+ }
+
+ /**
+ * @brief Copy constructor for weight
+ *
+ * @param rhs weight to construct from
+ */
+ Weight(const Weight &rhs);
+
+ /**
+ * @brief Move constructor for weight
+ *
+ * @param rhs weight to construct from
+ */
+ Weight(Weight &&rhs) = default;
+
+ /**
+ * @brief copy assigment
+ *
+ * @param rhs copy from
+ * @return Weight& Updated weight
+ */
+ Weight &operator=(const Weight &rhs);
+
+ /**
+ * @brief move assignment
+ *
+ * @param rhs move from
+ * @return Weight& Updated weight
+ */
+ Weight &operator=(Weight &&rhs) = default;
+
+ /**
+ * @brief Get the TensorDim
+ *
+ * @return TensorDim Dimension
+ */
+ TensorDim getDim() { return var.getDim(); }
+
+ /**
+ * @brief Get if the weight is trainable
+ *
+ * @return true if trainable
+ * @return false is not trainable
+ */
+ bool getTrainable() { return trainable; }
+
+ /**
+ * @brief Get the name of the weight
+ *
+ * @return std::string name
+ */
+ std::string getName() { return name; }
+
+ /**
+ * @brief Get the variable tensor (by name)
+ *
+ * @return Tensor Variable tensor
+ */
+ Tensor getVariable() { return var; }
+
+ /**
+ * @brief Get the Gradient tensor (by name)
+ *
+ * @return Tensor Gradient tensor
+ */
+ Tensor getGradient() { return grad; }
+
+private:
+ /**
+ * @brief Get the variable tensor (by reference)
+ *
+ * @return Tensor Variable tensor
+ */
+ Tensor &getVariableRef() { return var; }
+
+ /**
+ * @brief Get the Gradient tensor (by reference)
+ *
+ * @return Tensor Gradient tensor
+ */
+ Tensor &getGradientRef() { return grad; }
+
+ Tensor var; /**< variable to be updated and used */
+ Tensor grad; /**< gradient for the variable */
+ WeightInitializer initializer; /**< initializer for this variable */
+ bool trainable; /**< if this variable is trainable */
+ std::string name; /**< name of the parameter */
+};
+
+} // namespace nntrainer
+
+#endif /** __WEIGHT_H__ */
--- /dev/null
+util_sources = [
+ 'parse_util.cpp',
+ 'util_func.cpp'
+]
+
+foreach s : util_sources
+ nntrainer_sources += join_paths(meson.current_source_dir(), s)
+endforeach
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file parse_util.cpp
+ * @date 08 April 2020
+ * @brief This is collection of math functions
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <array>
+#include <assert.h>
+#include <cstring>
+#include <iostream>
+#include <layer_internal.h>
+#include <neuralnet.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <optimizer_internal.h>
+#include <parse_util.h>
+#include <pooling2d_layer.h>
+#include <sstream>
+#include <string>
+
+#define NUM_SKIP_CHAR 3
+
+namespace nntrainer {
+
+int getKeyValue(std::string input_str, std::string &key, std::string &value) {
+ int status = ML_ERROR_NONE;
+ std::vector<std::string> list;
+ std::regex words_regex("[^\\s=]+");
+ input_str.erase(std::remove(input_str.begin(), input_str.end(), ' '),
+ input_str.end());
+ auto words_begin =
+ std::sregex_iterator(input_str.begin(), input_str.end(), words_regex);
+ auto words_end = std::sregex_iterator();
+ int nwords = std::distance(words_begin, words_end);
+ if (nwords != 2) {
+ ml_loge("Error: input string must be 'key = value' format");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ for (std::sregex_iterator i = words_begin; i != words_end; ++i) {
+ list.push_back((*i).str());
+ }
+
+ key = list[0];
+ value = list[1];
+ return status;
+}
+
+unsigned int parseType(std::string ll, InputType t) {
+ unsigned int ret;
+ unsigned int i;
+ /**
+ * @brief Optimizer String from configure file
+ * "adam" : Adaptive Moment Estimation
+ * "sgd" : Stochestic Gradient Descent
+ */
+ std::array<std::string, 2> optimizer_string = {"adam", "sgd"};
+
+ /**
+ * @brief Loss Function String from configure file
+ * "mse" : Mean Squared Error
+ * "caterogical" : Categorical Cross Entropy
+ */
+ std::array<std::string, 2> loss_string = {"mse", "cross"};
+
+ /**
+ * @brief Model Type String from configure file
+ * "knn" : K Neearest Neighbor
+ * "regression" : Logistic Regression
+ * "neuralnet" : Neural Network
+ */
+ std::array<std::string, 3> model_type_string = {"knn", "regression",
+ "neuralnet"};
+
+ /**
+ * @brief Activation Type String from configure file
+ * "tanh" : tanh
+ * "sigmoid" : sigmoid
+ * "relu" : relu
+ * "softmax" : softmax
+ * "none" : none
+ * "unknown" : unknown
+ */
+ std::array<std::string, 6> activation_string = {
+ "tanh", "sigmoid", "relu", "softmax", "none", "unknown"};
+
+ /**
+ * @brief Layer Type String from configure file
+ * "input" : Input Layer Object
+ * "fully_conntected" : Fully Connected Layer Object
+ * "batch_normalization" : Batch Normalization Layer Object
+ * "conv2d" : Convolution 2D Layer Object
+ * "pooling2d" : Pooling 2D Layer Object
+ * "flatten" : Flatten Layer Object
+ * "activation" : Activation Layer Object
+ * "addition" : Addition Layer Object
+ */
+ std::array<std::string, 8> layer_string = {
+ "input", "fully_connected", "batch_normalization", "conv2d",
+ "pooling2d", "flatten", "activation", "addition"};
+
+ /**
+ * @brief Weight Initialization Type String from configure file
+ * "zeros" : Zero Initialization
+ * "ones" : One Initialization
+ * "lecun_normal" : LeCun Normal Initialization
+ * "lecun_uniform" : LeCun Uniform Initialization
+ * "xavier_normal" : Xavier Normal Initialization
+ * "xavier_uniform" : Xavier Uniform Initialization
+ * "he_normal" : He Normal Initialization
+ * "he_uniform" : He Uniform Initialization
+ */
+ std::array<std::string, 8> weight_ini_string = {
+ "zeros", "ones", "lecun_normal", "lecun_uniform",
+ "xavier_normal", "xavier_uniform", "he_normal", "he_uniform"};
+
+ /**
+ * @brief Weight Decay String from configure file
+ * "L2Norm" : squared norm regularization
+ * "Regression" : Regression
+ */
+ std::array<std::string, 2> weight_regularizer_string = {"l2norm",
+ "regression"};
+
+ /**
+ * @brief Weight Decay String from configure file
+ * "L2Norm" : squared norm regularization
+ * "Regression" : Regression
+ */
+ std::array<std::string, 3> padding_string = {"full", "same", "valid"};
+
+ /**
+ * @brief Pooling String from configure file
+ * "max" : Max Pooling
+ * "average" : Average Pooling
+ * "global_max" : Global Max Pooling
+ * "global_average" : Global Average Pooling
+ */
+ std::array<std::string, 4> pooling_string = {"max", "average", "global_max",
+ "global_average"};
+
+ switch (t) {
+ case TOKEN_OPT:
+ for (i = 0; i < optimizer_string.size(); i++) {
+ if (!strncasecmp(optimizer_string[i].c_str(), ll.c_str(),
+ optimizer_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)OptType::UNKNOWN;
+ break;
+ case TOKEN_LOSS:
+ for (i = 0; i < loss_string.size(); i++) {
+ if (!strncasecmp(loss_string[i].c_str(), ll.c_str(),
+ loss_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)LossType::LOSS_UNKNOWN;
+ break;
+ case TOKEN_MODEL:
+ for (i = 0; i < model_type_string.size(); i++) {
+ if (!strncasecmp(model_type_string[i].c_str(), ll.c_str(),
+ model_type_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)NetType::UNKNOWN;
+ break;
+ case TOKEN_ACTI:
+ for (i = 0; i < activation_string.size(); i++) {
+ if (!strncasecmp(activation_string[i].c_str(), ll.c_str(),
+ activation_string[i].size())) {
+
+ return (i);
+ }
+ }
+ ml_logw("Input activation %s cannot be identified. "
+ "Moved to NO activation layer by default.",
+ ll.c_str());
+ ret = (unsigned int)ActivationType::ACT_UNKNOWN;
+ break;
+ case TOKEN_LAYER:
+ for (i = 0; i < layer_string.size(); i++) {
+ if (!strncasecmp(layer_string[i].c_str(), ll.c_str(),
+ layer_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)LayerType::LAYER_UNKNOWN;
+ break;
+ case TOKEN_WEIGHT_INIT:
+ for (i = 0; i < weight_ini_string.size(); i++) {
+ if (!strncasecmp(weight_ini_string[i].c_str(), ll.c_str(),
+ weight_ini_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)WeightInitializer::WEIGHT_UNKNOWN;
+ break;
+ case TOKEN_WEIGHT_REGULARIZER:
+ for (i = 0; i < weight_regularizer_string.size(); i++) {
+ if (!strncasecmp(weight_regularizer_string[i].c_str(), ll.c_str(),
+ weight_regularizer_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)WeightRegularizerType::unknown;
+ break;
+ case TOKEN_PADDING:
+ for (i = 0; i < padding_string.size(); i++) {
+ if (!strncasecmp(padding_string[i].c_str(), ll.c_str(),
+ padding_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)Pooling2DLayer::PaddingType::unknown;
+ break;
+ case TOKEN_POOLING:
+ for (i = 0; i < pooling_string.size(); i++) {
+ if (!strncasecmp(pooling_string[i].c_str(), ll.c_str(),
+ pooling_string[i].size())) {
+ return (i);
+ }
+ }
+ ret = (unsigned int)Pooling2DLayer::PoolingType::unknown;
+ break;
+ case TOKEN_UNKNOWN:
+ default:
+ ml_loge("Error: unknown token cannot be parsed.");
+ ret = 0;
+ break;
+ }
+ return ret;
+}
+
+/**
+ * @brief Layer Properties
+ * input_shape = 0
+ * normalization = 1
+ * standardization = 2
+ * activation = 3
+ * epsilon = 4
+ * weight_regularizer = 5
+ * weight_regularizer_constant = 6
+ * unit = 7
+ * weight_initializer = 8
+ * bias_initializer = 9
+ * filters = 10
+ * kernel_size = 11
+ * stride = 12
+ * padding = 13
+ * pool_size = 14
+ * pooling = 15
+ * flatten = 16
+ * name = 17
+ * num_inputs = 18
+ * num_outputs = 19
+ * momentum = 20
+ * moving_mean_initializer = 21
+ * moving_variance_initializer = 22
+ * gamma_initializer = 23
+ * beta_initializer = 24
+ * input_layers = 25
+ *
+ * InputLayer has 0, 1, 2, 3 properties.
+ * FullyConnectedLayer has 1, 4, 6, 7, 8, 9 properties.
+ * Conv2DLayer has 0, 1, 4, 6, 7, 9, 10, 11, 12, 13 properties.
+ * Pooling2DLayer has 12, 13, 14, 15 properties.
+ * BatchNormalizationLayer has 0, 1, 5, 6, 7 properties.
+ */
+static std::array<std::string, 28> property_string = {
+ "input_shape",
+ "normalization",
+ "standardization",
+ "activation",
+ "epsilon",
+ "weight_regularizer",
+ "weight_regularizer_constant",
+ "unit",
+ "weight_initializer",
+ "bias_initializer",
+ "filters",
+ "kernel_size",
+ "stride",
+ "padding",
+ "pool_size",
+ "pooling",
+ "flatten",
+ "name",
+ "num_inputs",
+ "num_outputs",
+ "momentum",
+ "moving_mean_initializer",
+ "moving_variance_initializer",
+ "gamma_initializer",
+ "beta_initializer",
+ "modelfile",
+ "input_layers",
+ "unknown"};
+
+unsigned int parseLayerProperty(std::string property) {
+ unsigned int i;
+
+ for (i = 0; i < property_string.size(); i++) {
+ unsigned int size = (property_string[i].size() > property.size())
+ ? property_string[i].size()
+ : property.size();
+
+ if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
+ return (i);
+ }
+ }
+
+ return (unsigned int)Layer::PropertyType::unknown;
+}
+
+std::string propToStr(unsigned int type) { return property_string[type]; }
+
+unsigned int parseOptProperty(std::string property) {
+ unsigned int i;
+
+ /**
+ * @brief Optimizer Properties
+ * learning_rate = 0,
+ * decay_rate = 1,
+ * decay_steps = 2
+ * beta1 = 3,
+ * beta2 = 4,
+ * epsilon = 5,
+ * continue_train = 6,
+ */
+ std::array<std::string, 7> property_string = {
+ "learning_rate", "decay_rate", "decay_steps", "beta1",
+ "beta2", "epsilon", "continue_train"};
+
+ for (i = 0; i < property_string.size(); i++) {
+ unsigned int size = (property_string[i].size() > property.size())
+ ? property_string[i].size()
+ : property.size();
+
+ if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
+ return (i);
+ }
+ }
+
+ return (unsigned int)Optimizer::PropertyType::unknown;
+}
+
+unsigned int parseNetProperty(std::string property) {
+ unsigned int i;
+
+ /**
+ * @brief Network Properties
+ * loss_val = 0,
+ * loss = 1,
+ * batch_size = 2,
+ * epochs = 3,
+ * save_path = 4
+ */
+ std::array<std::string, 5> property_string = {
+ "loss_val", "loss", "batch_size", "epochs", "save_path"};
+
+ for (i = 0; i < property_string.size(); i++) {
+ unsigned int size = (property_string[i].size() > property.size())
+ ? property_string[i].size()
+ : property.size();
+
+ if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
+ return (i);
+ }
+ }
+
+ return (unsigned int)NeuralNetwork::PropertyType::unknown;
+}
+
+unsigned int parseDataProperty(std::string property) {
+ unsigned int i;
+
+ /**
+ * @brief Data Properties
+ * train_data = 0,
+ * val_data = 1,
+ * test_data = 2,
+ * label_data = 3,
+ * buffer_size = 4
+ */
+ std::array<std::string, 5> property_string = {
+ "train_data", "val_data", "test_data", "label_data", "buffer_size"};
+
+ for (i = 0; i < property_string.size(); i++) {
+ unsigned int size = (property_string[i].size() > property.size())
+ ? property_string[i].size()
+ : property.size();
+
+ if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
+ return (i);
+ }
+ }
+
+ return (unsigned int)DataBuffer::PropertyType::unknown;
+}
+
+int setUint(unsigned int &val, const std::string &str) {
+ int status = ML_ERROR_NONE;
+ try {
+ val = (unsigned int)std::stoul(str.c_str());
+ } catch (...) {
+ ml_loge("Error: Wrong Type. Must be int");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return status;
+}
+
+int setFloat(float &val, std::string str) {
+ int status = ML_ERROR_NONE;
+ try {
+ val = std::stof(str.c_str());
+ } catch (...) {
+ ml_loge("Error: Wrong Type. Must be float");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+ return status;
+}
+
+int setDouble(double &val, std::string str) {
+ int status = ML_ERROR_NONE;
+ try {
+ val = std::stod(str.c_str());
+ } catch (...) {
+ ml_loge("Error: Wrong Type. Must be double");
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return status;
+}
+
+int setBoolean(bool &val, std::string str) {
+ int status = ML_ERROR_NONE;
+ std::string t = "true";
+ std::string f = "false";
+
+ if (!strncasecmp(str.c_str(), t.c_str(), t.size())) {
+ val = true;
+ } else if (!strncasecmp(str.c_str(), f.c_str(), f.size())) {
+ val = false;
+ } else {
+ status = ML_ERROR_INVALID_PARAMETER;
+ }
+
+ return status;
+}
+
+int getValues(int n_str, std::string str, int *value) {
+ int status = ML_ERROR_NONE;
+ std::regex words_regex("[^\\s.,:;!?]+");
+ str.erase(std::remove(str.begin(), str.end(), ' '), str.end());
+ auto words_begin = std::sregex_iterator(str.begin(), str.end(), words_regex);
+ auto words_end = std::sregex_iterator();
+
+ int num = std::distance(words_begin, words_end);
+ if (num != n_str) {
+ ml_loge("Number of Data is not match");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+ int cn = 0;
+ for (std::sregex_iterator i = words_begin; i != words_end; ++i) {
+ value[cn] = std::stoi((*i).str());
+ cn++;
+ }
+ return status;
+}
+
+const char *getValues(std::vector<int> values, const char *delimiter) {
+ std::stringstream vec_str;
+
+ if (values.empty())
+ return "unknown";
+
+ std::copy(values.begin(), values.end() - 1,
+ std::ostream_iterator<int>(vec_str, delimiter));
+ vec_str << values.back();
+
+ return std::move(vec_str.str().c_str());
+}
+
+std::vector<std::string> split(const std::string &s, std::regex ®) {
+ std::vector<std::string> out;
+ char char_to_remove[NUM_SKIP_CHAR] = {' ', '[', ']'};
+ std::string str = s;
+ for (unsigned int i = 0; i < NUM_SKIP_CHAR; ++i) {
+ str.erase(std::remove(str.begin(), str.end(), char_to_remove[i]),
+ str.end());
+ }
+ std::regex_token_iterator<std::string::iterator> end;
+ std::regex_token_iterator<std::string::iterator> iter(str.begin(), str.end(),
+ reg, -1);
+
+ while (iter != end) {
+ out.push_back(*iter);
+ ++iter;
+ }
+ return out;
+}
+
+} /* namespace nntrainer */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file parse_util.h
+ * @date 07 May 2020
+ * @brief This is collection of parse functions.
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __PARSE_UTIL_H__
+#define __PARSE_UTIL_H__
+#ifdef __cplusplus
+
+#include <iostream>
+#include <regex>
+#include <string>
+#include <vector>
+
+namespace nntrainer {
+
+#define NN_RETURN_STATUS() \
+ do { \
+ if (status != ML_ERROR_NONE) { \
+ return status; \
+ } \
+ } while (0)
+
+/**
+ * @brief Enumeration for input configuration file parsing
+ * 0. OPT ( Optimizer Token )
+ * 1. LOSS ( Loss Function Token )
+ * 2. MODEL ( Model Token )
+ * 3. ACTI ( Activation Token )
+ * 4. LAYER ( Layer Token )
+ * 5. WEIGHT_INIT ( Weight Initializer Token )
+ * 7. WEIGHT_REGULARIZER ( Weight Decay Token )
+ * 8. PADDING ( Padding Token )
+ * 9. POOLING ( Pooling Token )
+ * 9. UNKNOWN
+ */
+typedef enum {
+ TOKEN_OPT,
+ TOKEN_LOSS,
+ TOKEN_MODEL,
+ TOKEN_ACTI,
+ TOKEN_LAYER,
+ TOKEN_WEIGHT_INIT,
+ TOKEN_WEIGHT_REGULARIZER,
+ TOKEN_PADDING,
+ TOKEN_POOLING,
+ TOKEN_UNKNOWN
+} InputType;
+
+inline void throw_status(int status) {
+ switch (status) {
+ case ML_ERROR_NONE:
+ break;
+ case ML_ERROR_INVALID_PARAMETER:
+ throw std::invalid_argument("invalid argument from c style throw");
+ case ML_ERROR_OUT_OF_MEMORY:
+ throw std::bad_alloc();
+ case ML_ERROR_TIMED_OUT:
+ throw std::runtime_error("Timed out from c style throw");
+ case ML_ERROR_PERMISSION_DENIED:
+ throw std::runtime_error("permission denied from c style throw");
+ case ML_ERROR_UNKNOWN:
+ default:
+ throw std::runtime_error("unknown error from c style throw");
+ }
+}
+/**
+ * @brief Parsing Layer Property
+ * @param[in] property string to be parsed
+ * @retval int enumerated type
+ */
+unsigned int parseLayerProperty(std::string property);
+
+/**
+ * @brief Unparse Layer property to string
+ * @param[in] type property type
+ * @retval string representation of the type
+ */
+std::string propToStr(const unsigned int type);
+
+/**
+ * @brief Parsing Configuration Token
+ * @param[in] ll string to be parsed
+ * @param[in] t Token type
+ * @retval int enumerated type
+ */
+unsigned int parseType(std::string ll, InputType t);
+
+/**
+ * @brief Parsing Optimizer Property
+ * @param[in] property string to be parsed
+ * @retval int enumerated type
+ */
+unsigned int parseOptProperty(std::string property);
+
+/**
+ * @brief Parsing Network Property
+ * @param[in] property string to be parsed
+ * @retval int enumerated type
+ */
+unsigned int parseNetProperty(std::string property);
+
+/**
+ * @brief Parsing Data Buffer Property
+ * @param[in] property string to be parsed
+ * @retval int enumerated type
+ */
+unsigned int parseDataProperty(std::string property);
+
+/**
+ * @brief check str to be unsigned int and assign to variable to type T
+ * @param[out] val assign variable
+ * @param[in] str input string
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+int setUint(unsigned int &val, const std::string &str);
+
+/**
+ * @brief check str to be float and assign
+ * @param[out] val assign variable
+ * @param[in] str input string
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+int setFloat(float &val, std::string str);
+
+/**
+ * @brief check str to be double and assign
+ * @param[out] val assign variable
+ * @param[in] str input string
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+int setDouble(double &val, std::string str);
+
+/**
+ * @brief check str to be bool and assign
+ * @param[out] val assign variable
+ * @param[in] str input string
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+int setBoolean(bool &val, std::string str);
+
+/**
+ * @brief parse string and return key & value
+ * @param[in] input_str input string to split with '='
+ * @param[out] key key
+ * @param[out] value value
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+int getKeyValue(std::string input_str, std::string &key, std::string &value);
+
+/**
+ * @brief join vector of int to string with delimiter ","
+ * @param[in] values vector of int
+ * @param[in] delimiter delimiter for the string
+ * @retval output string
+ */
+const char *getValues(std::vector<int> values, const char *delimiter = ",");
+
+int getValues(int n_str, std::string str, int *value);
+
+/**
+ * @brief split string into vector with delimiter regex
+ * @param[in] str string
+ * @param[in] reg regular expression to use as delimiter
+ * @retval output string vector
+ */
+std::vector<std::string> split(const std::string &s, std::regex ®);
+
+/**
+ * @brief print instance info. as <Type at (address)>
+ * @param[in] std::ostream &out, T&& t
+ * @param[in] t pointer to the instance
+ */
+template <typename T,
+ typename std::enable_if_t<std::is_pointer<T>::value, T> * = nullptr>
+void printInstance(std::ostream &out, const T &t) {
+ out << '<' << typeid(*t).name() << " at " << t << '>' << std::endl;
+}
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __PARSE_UTIL_H__ */
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file util_func.cpp
+ * @date 08 April 2020
+ * @brief This is collection of math functions
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#include <cmath>
+#include <fstream>
+#include <random>
+
+#include <util_func.h>
+
+namespace nntrainer {
+
+static auto rng = [] {
+ std::mt19937 rng;
+ rng.seed(getSeed());
+ return rng;
+}();
+static std::uniform_real_distribution<float> dist(-0.5, 0.5);
+
+unsigned int getSeed() { return 0; }
+
+float random() { return dist(rng); }
+
+float sqrtFloat(float x) { return sqrt(x); };
+
+double sqrtDouble(double x) { return sqrt(x); };
+
+float logFloat(float x) { return log(x + 1.0e-20); }
+
+float exp_util(float x) { return exp(x); }
+
+// This is 2D zero pad
+// TODO : Optimize for multi dimention padding
+Tensor zero_pad(int batch, Tensor const &in, unsigned int const *padding) {
+ unsigned int c = in.channel();
+ unsigned int h = in.height();
+ unsigned int w = in.width();
+
+ unsigned int height_p = h + padding[0] * 2;
+ unsigned int width_p = w + padding[1] * 2;
+
+ unsigned int height_p_h = h + padding[0];
+ unsigned int width_p_h = w + padding[1];
+
+ Tensor output(1, c, height_p, width_p);
+ output.setZero();
+
+ for (unsigned int j = 0; j < c; ++j) {
+ for (unsigned int k = 0; k < padding[0]; ++k) {
+ for (unsigned int l = 0; l < width_p; ++l) {
+ output.setValue(0, j, k, l, 0.0f);
+ output.setValue(0, j, k + height_p_h, l, 0.0f);
+ }
+ }
+
+ for (unsigned int l = 0; l < padding[1]; ++l) {
+ for (unsigned int k = padding[0]; k < h; ++k) {
+ output.setValue(0, j, k, l, 0.0f);
+ output.setValue(0, j, k, l + width_p_h, 0.0f);
+ }
+ }
+ }
+
+ for (unsigned int j = 0; j < c; ++j) {
+ for (unsigned int k = 0; k < h; ++k) {
+ for (unsigned int l = 0; l < w; ++l) {
+ output.setValue(0, j, k + padding[0], l + padding[1],
+ in.getValue(batch, j, k, l));
+ }
+ }
+ }
+
+ return output;
+}
+
+// This is strip pad and return original tensor
+Tensor strip_pad(Tensor const &in, unsigned int const *padding) {
+ Tensor output(in.batch(), in.channel(), in.height() - padding[0] * 2,
+ in.width() - padding[1] * 2);
+ output.setZero();
+
+ for (unsigned int i = 0; i < in.batch(); ++i) {
+ for (unsigned int j = 0; j < in.channel(); ++j) {
+ for (unsigned int k = 0; k < output.height(); ++k) {
+ for (unsigned int l = 0; l < output.width(); ++l) {
+ output.setValue(i, j, k, l,
+ in.getValue(i, j, k + padding[0], l + padding[1]));
+ }
+ }
+ }
+ }
+ return output;
+}
+
+Tensor rotate_180(Tensor in) {
+ Tensor output(in.getDim());
+ output.setZero();
+ for (unsigned int i = 0; i < in.batch(); ++i) {
+ for (unsigned int j = 0; j < in.channel(); ++j) {
+ for (unsigned int k = 0; k < in.height(); ++k) {
+ for (unsigned int l = 0; l < in.width(); ++l) {
+ output.setValue(
+ i, j, k, l,
+ in.getValue(i, j, (in.height() - k - 1), (in.width() - l - 1)));
+ }
+ }
+ }
+ }
+ return output;
+}
+
+bool isFileExist(std::string file_name) {
+ std::ifstream infile(file_name);
+ return infile.good();
+}
+
+} // namespace nntrainer
--- /dev/null
+/**
+ * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @file util_func.h
+ * @date 08 April 2020
+ * @brief This is collection of math functions.
+ * @see https://github.com/nnstreamer/nntrainer
+ * @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @bug No known bugs except for NYI items
+ *
+ */
+
+#ifndef __UTIL_FUNC_H__
+#define __UTIL_FUNC_H__
+#ifdef __cplusplus
+
+#include <tensor.h>
+
+namespace nntrainer {
+
+/**
+ * @brief get the seed
+ * @retVal seed
+ */
+unsigned int getSeed();
+
+/**
+ * @brief random function
+ */
+float random();
+
+/**
+ * @brief sqrt function for float type
+ * @param[in] x float
+ */
+float sqrtFloat(float x);
+
+double sqrtDouble(double x);
+
+/**
+ * @brief log function for float type
+ * @param[in] x float
+ */
+float logFloat(float x);
+
+/**
+ * @brief exp function for float type
+ * @param[in] x float
+ */
+float exp_util(float x);
+
+/**
+ * @brief apply padding
+ * @param[in] batch batch index
+ * @param[in] x input
+ * @param[in] padding 2D padding size
+ * @retVal Tensor output tensor with batch size is 1 for batch index
+ */
+Tensor zero_pad(int batch, Tensor const &in, unsigned int const *padding);
+
+/**
+ * @brief strip padding
+ * @param[in] x input
+ * @param[in] padding 2D padding size
+ * @retVal Tensor output tensor without padding
+ */
+Tensor strip_pad(Tensor const &in, unsigned int const *padding);
+
+/**
+ * @brief rotate 180 dgree
+ * @param[in] in input Tensor
+ * @retVal Tensor rotated tensor (180 degree)
+ */
+Tensor rotate_180(Tensor in);
+
+/**
+ * @brief Check Existance of File
+ * @param[in] file path of the file to be checked
+ * @returns true if file exists, else false
+ */
+bool isFileExist(std::string file);
+
+} /* namespace nntrainer */
+
+#endif /* __cplusplus */
+#endif /* __UTIL_FUNC_H__ */
%files devel
%{_includedir}/nntrainer/databuffer.h
%{_includedir}/nntrainer/databuffer_factory.h
-%{_includedir}/nntrainer/databuffer_file.h
-%{_includedir}/nntrainer/databuffer_func.h
-%{_includedir}/nntrainer/databuffer_util.h
%{_includedir}/nntrainer/layer_internal.h
%{_includedir}/nntrainer/layer_factory.h
-%{_includedir}/nntrainer/input_layer.h
-%{_includedir}/nntrainer/fc_layer.h
-%{_includedir}/nntrainer/bn_layer.h
-%{_includedir}/nntrainer/conv2d_layer.h
-%{_includedir}/nntrainer/pooling2d_layer.h
-%{_includedir}/nntrainer/flatten_layer.h
-%{_includedir}/nntrainer/loss_layer.h
-%{_includedir}/nntrainer/activation_layer.h
%{_includedir}/nntrainer/neuralnet.h
-%{_includedir}/nntrainer/model_loader.h
%{_includedir}/nntrainer/tensor.h
-%{_includedir}/nntrainer/lazy_tensor.h
%{_includedir}/nntrainer/tensor_dim.h
-%{_includedir}/nntrainer/nntrainer_log.h
-%{_includedir}/nntrainer/nntrainer_logger.h
%{_includedir}/nntrainer/optimizer_internal.h
-%{_includedir}/nntrainer/util_func.h
-%{_includedir}/nntrainer/parse_util.h
-%{_includedir}/nntrainer/addition_layer.h
-%{_includedir}/nntrainer/concat_layer.h
-%{_includedir}/nntrainer/delegate.h
+%{_includedir}/nntrainer/optimizer_factory.h
%{_includedir}/nntrainer/nntrainer-api-common.h
-%{_includedir}/nntrainer/blas_interface.h
%{_includedir}/nntrainer/weight.h
-%{_includedir}/nntrainer/adam.h
-%{_includedir}/nntrainer/sgd.h
-%{_includedir}/nntrainer/optimizer_factory.h
-%if 0%{?support_nnstreamer_backbone}
-%{_includedir}/nntrainer/nnstreamer_layer.h
-%endif
-%{_includedir}/nntrainer/tflite_layer.h
%{_libdir}/pkgconfig/nntrainer.pc
%files devel-static
"padding = 0,0 |");
static int SUCCESS = 0;
-static int LOADFAIL = initest::LOAD;
-static int INITFAIL = initest::INIT;
-static int ALLFAIL = LOADFAIL | INITFAIL;
using I = IniSection;
projects / platform / core / ml / nntrainer.git / commitdiff