include $(CLEAR_VARS)
+LOCAL_MODULE := ccapi-nntrainer
+LOCAL_SRC_FILES := $(NNTRAINER_ROOT)/libs/$(TARGET_ARCH_ABI)/libccapi-nntrainer.so
+
+include $(PREBUILT_SHARED_LIBRARY)
+
+include $(CLEAR_VARS)
+
LOCAL_ARM_NEON := true
LOCAL_CFLAGS += -std=c++17 -Ofast -mcpu=cortex-a53 -Ilz4-nougat/lib
LOCAL_LDFLAGS += -Llz4-nougat/lib/obj/local/$(TARGET_ARCH_ABI)/
LOCAL_SRC_FILES := main.cpp
-LOCAL_SHARED_LIBRARIES := nntrainer
+LOCAL_SHARED_LIBRARIES := nntrainer ccapi-nntrainer
LOCAL_C_INCLUDES += $(NNTRAINER_INCLUDES)
#include <time.h>
#include <databuffer.h>
-#include <databuffer_func.h>
#include <neuralnet.h>
#include <tensor.h>
srand(time(NULL));
- auto data_train = std::make_shared<nntrainer::DataBufferFromCallback>();
- data_train->setGeneratorFunc(getBatch_train);
+ auto data_train =
+ ml::train::createDataset(ml::train::DatasetType::GENERATOR, getBatch_train);
/**
* @brief Create NN
}
if (training) {
- NN.setDataBuffer(ml::train::DatasetDataUsageType::DATA_TRAIN, data_train);
+ NN.setDataset(ml::train::DatasetDataUsageType::DATA_TRAIN,
+ std::move(data_train));
try {
NN.train();
e = executable('nntrainer_logistic',
'main.cpp',
- dependencies: [iniparser_dep, nntrainer_dep],
+ dependencies: [iniparser_dep, nntrainer_dep, nntrainer_ccapi_dep],
install: get_option('install-app'),
install_dir: application_install_dir
)
include $(CLEAR_VARS)
+LOCAL_MODULE := ccapi-nntrainer
+LOCAL_SRC_FILES := $(NNTRAINER_ROOT)/libs/$(TARGET_ARCH_ABI)/libccapi-nntrainer.so
+
+include $(PREBUILT_SHARED_LIBRARY)
+
+include $(CLEAR_VARS)
+
LOCAL_MODULE := app_utils
LOCAL_SRC_FILES := $(NNTRAINER_ROOT)/Applications/utils/libs/$(TARGET_ARCH_ABI)/libapp_utils.so
APP_UTILS_INCLUDES := $(NNTRAINER_ROOT)/Applications/utils/jni/includes
LOCAL_SRC_FILES := main.cpp
-LOCAL_SHARED_LIBRARIES := nntrainer app_utils
+LOCAL_SHARED_LIBRARIES := nntrainer app_utils ccapi-nntrainer
LOCAL_C_INCLUDES += $(NNTRAINER_INCLUDES) $(APP_UTILS_INCLUDES)
#include <iostream>
#include <sstream>
#include <stdlib.h>
-
-#include "databuffer.h"
-#include "databuffer_func.h"
-#include "neuralnet.h"
-#include "nntrainer_error.h"
-#include "tensor.h"
#include <vector>
+#include <databuffer.h>
+#include <neuralnet.h>
+#include <nntrainer_error.h>
+#include <tensor.h>
+
/**
* @brief Data size for each category
*/
for (unsigned int i = 0; i < count_val.remain; ++i)
count_val.duplication[i] = i;
- auto db_train = std::make_shared<nntrainer::DataBufferFromCallback>();
- db_train->setGeneratorFunc(getBatch_train_file);
- auto db_valid = std::make_shared<nntrainer::DataBufferFromCallback>();
- db_valid->setGeneratorFunc(getBatch_val_file);
+ auto db_train = ml::train::createDataset(ml::train::DatasetType::GENERATOR,
+ getBatch_train_file);
+ auto db_valid = ml::train::createDataset(ml::train::DatasetType::GENERATOR,
+ getBatch_val_file);
/**
* @brief Neural Network Create & Initialization
try {
NN.readModel();
- NN.setDataBuffer(ml::train::DatasetDataUsageType::DATA_TRAIN, db_train);
- NN.setDataBuffer(ml::train::DatasetDataUsageType::DATA_VAL, db_valid);
+ NN.setDataset(ml::train::DatasetDataUsageType::DATA_TRAIN,
+ std::move(db_train));
+ NN.setDataset(ml::train::DatasetDataUsageType::DATA_VAL,
+ std::move(db_valid));
NN.train();
training_loss = NN.getTrainingLoss();
validation_loss = NN.getValidationLoss();
executable('nntrainer_vgg',
mnist_sources,
- dependencies: [iniparser_dep, nntrainer_dep, app_utils_dep],
+ dependencies: [iniparser_dep, nntrainer_dep, nntrainer_ccapi_dep, app_utils_dep],
include_directories: include_directories('.'),
install: get_option('install-app'),
install_dir: application_install_dir
$(NNTRAINER_ROOT)/nntrainer/dataset/batch_queue.cpp \
$(NNTRAINER_ROOT)/nntrainer/dataset/databuffer.cpp \
$(NNTRAINER_ROOT)/nntrainer/dataset/databuffer_factory.cpp \
- $(NNTRAINER_ROOT)/nntrainer/dataset/databuffer_func.cpp \
- $(NNTRAINER_ROOT)/nntrainer/dataset/databuffer_file.cpp \
$(NNTRAINER_ROOT)/nntrainer/dataset/func_data_producer.cpp \
$(NNTRAINER_ROOT)/nntrainer/dataset/random_data_producers.cpp \
$(NNTRAINER_ROOT)/nntrainer/dataset/raw_file_data_producer.cpp \
constexpr char USER_DATA[] = "user_data";
-DataBuffer::DataBuffer(DatasetType type) :
- data_buffer_type(type),
- batch_producer(),
- data_q(),
- label_q(),
- queue_status(DataStatus::DATA_NOT_READY),
- buf_size(0),
- cur_bufsize(0),
- class_num(0),
- batch_size(0),
- samples_per_epoch(0),
- remaining_samples_per_epoch(0),
- is_running(false),
- initialized(false),
- producer(nullptr),
- db_props(new Props()) {
- rng.seed(getSeed());
-};
-
DataBuffer::DataBuffer(std::unique_ptr<DataProducer> &&producer_) :
producer(std::move(producer_)),
- db_props(new Props()) {
- rng.seed(getSeed());
-}
+ db_props(new Props()) {}
DataBuffer::~DataBuffer(){};
-int DataBuffer::rangeRandom(int min, int max) {
- std::uniform_int_distribution<int> dist(min, max);
- return dist(rng);
-}
-
-int DataBuffer::run() {
- if (queue_status == DataStatus::DATA_ERROR)
- return ML_ERROR_INVALID_PARAMETER;
-
- if (initialized) {
- this->is_running = true;
- this->batch_producer = std::thread(&DataBuffer::updateData, this);
- if (consumer_exception_ptr) {
- try {
- std::rethrow_exception(consumer_exception_ptr);
- } catch (const std::exception &ex) {
- std::cout << ex.what() << "\n";
- return ML_ERROR_INVALID_PARAMETER;
- }
- }
- } else {
- ml_loge("Error: Training Data Set is not valid");
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- return ML_ERROR_NONE;
-}
-
-int DataBuffer::clear() {
- int status = ML_ERROR_NONE;
- setStateAndNotify(DataStatus::DATA_NOT_READY);
- is_running = false;
- if (initialized && true == batch_producer.joinable())
- batch_producer.join();
- this->data_q.clear();
- this->label_q.clear();
- this->cur_bufsize = 0;
- this->remaining_samples_per_epoch = samples_per_epoch;
- return status;
-}
-
-bool DataBuffer::getDataFromBuffer(float *out, float *label) {
- using QueueType = std::vector<std::vector<float>>;
-
- auto wait_for_data_fill = [](std::mutex &ready_mutex,
- std::condition_variable &cv, DataStatus &flag,
- const unsigned int batch_size,
- QueueType &queue) {
- while (true) {
- std::unique_lock<std::mutex> ul(ready_mutex);
- cv.wait(ul, [&]() -> bool { return flag != DataStatus::DATA_NOT_READY; });
- if (flag == DataStatus::DATA_ERROR || flag == DataStatus::DATA_END)
- return queue.size() < batch_size ? false : true;
-
- if (flag == DataStatus::DATA_READY && queue.size() >= batch_size)
- return true;
- }
-
- throw std::logic_error("[getDataFromBuffer] control should not reach here");
- };
-
- auto fill_bundled_data_from_queue =
- [](std::mutex &q_lock, QueueType &q, const unsigned int batch_size,
- const unsigned int feature_size, float *buf) {
- for (unsigned int b = 0; b < batch_size; ++b)
- std::copy(q[b].begin(), q[b].begin() + feature_size,
- buf + b * feature_size);
-
- q_lock.lock();
- q.erase(q.begin(), q.begin() + batch_size);
- q_lock.unlock();
- };
-
- /// facade that wait for the databuffer to be filled and pass it to outparam
- /// note that batch_size is passed as an argument because it can vary by
- /// DatasetDataUsageType::BUF_TYPE later...
- auto fill_out_params =
- [&](std::mutex &ready_mutex, std::condition_variable &cv, DataStatus &flag,
- QueueType &data_q, QueueType &label_q, const unsigned int batch_size,
- unsigned int &cur_bufsize) {
- if (!wait_for_data_fill(ready_mutex, cv, flag, batch_size, data_q)) {
- return false;
- }
-
- fill_bundled_data_from_queue(data_lock, data_q, batch_size,
- this->input_dim.getFeatureLen(), out);
- fill_bundled_data_from_queue(data_lock, label_q, batch_size,
- this->class_num, label);
-
- cur_bufsize -= batch_size;
- return true;
- };
-
- return fill_out_params(status_mutex, cv_status, queue_status, data_q, label_q,
- batch_size, cur_bufsize);
-}
-
-int DataBuffer::setClassNum(unsigned int num) {
- int status = ML_ERROR_NONE;
- if (num == 0) {
- ml_loge("Error: number of class should be bigger than 0");
- initialized = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
- class_num = num;
- return status;
-}
-
-int DataBuffer::setBatchSize(unsigned int size) {
- int status = ML_ERROR_NONE;
- if (size == 0) {
- ml_loge("Error: batch size must be greater than 0");
- initialized = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
- batch_size = size;
- return status;
-}
-
-int DataBuffer::init() {
- if (batch_size == 0) {
- ml_loge("Error: batch size must be greater than 0");
- initialized = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- if (buf_size < batch_size) {
- if (buf_size > 1) {
- ml_logw("Dataset buffer size reset to be at least batch size");
- }
- buf_size = batch_size;
- }
-
- if (!class_num) {
- ml_loge("Error: number of class must be set");
- initialized = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- if (!this->input_dim.getFeatureLen()) {
- ml_loge("Error: feature size must be set");
- initialized = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
-
- this->cur_bufsize = 0;
-
- setStateAndNotify(DataStatus::DATA_NOT_READY);
- return ML_ERROR_NONE;
-}
-
std::future<std::shared_ptr<BatchQueue>>
DataBuffer::startFetchWorker(const std::vector<TensorDim> &input_dims,
const std::vector<TensorDim> &label_dims) {
return producer->finalize(input_dims, label_dims);
}
-int DataBuffer::setFeatureSize(const TensorDim &indim) {
- int status = ML_ERROR_NONE;
- input_dim = indim;
- return status;
-}
-
void DataBuffer::displayProgress(const int count, float loss) {
int barWidth = 20;
+ /** this is temporary measure, will be getting this as an argument */
+ int batch_size = 1;
+ int samples_per_epoch = 0;
std::stringstream ssInt;
ssInt << count * batch_size;
<< "[DataBuffer] Failed to set property";
}
}
-
-int DataBuffer::setGeneratorFunc(datagen_cb func, void *user_data) {
- return ML_ERROR_NOT_SUPPORTED;
-}
-
-int DataBuffer::setDataFile(const std::string &path) {
- return ML_ERROR_NOT_SUPPORTED;
-}
-
-void DataBuffer::setStateAndNotify(const DataStatus status) {
- std::lock_guard<std::mutex> lgtrain(status_mutex);
- queue_status = status;
- cv_status.notify_all();
-}
-
} /* namespace nntrainer */
class DataBuffer : public ml::train::Dataset {
public:
/**
- * @brief Create Buffer
- * @retval DataBuffer
- */
- DataBuffer(DatasetType type);
-
- /**
* @brief Create DataBuffer with a producer
*
*/
* @brief Destroy the Data Buffer object
*
*/
- virtual ~DataBuffer();
-
- /**
- * @brief Initialize Buffer with data buffer private variables
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int init();
+ ~DataBuffer();
/**
* @brief prepare iteration a head of time with a dedicated worker. The
const std::vector<TensorDim> &label_dims);
/**
- * @brief Update Data Buffer ( it is for child thread )
- * @retval void
- */
- virtual void updateData(){};
-
- /**
- * @brief function for thread ( training, validation, test )
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int run();
-
- /**
- * @brief clear thread ( training, validation, test )
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int clear();
-
- /**
- * @brief get Data from Data Buffer using databuffer param
- * @param[out] out feature data ( batch_size size ), a contiguous and
- * allocated memory block should be passed
- * @param[out] label label data ( batch_size size ), a contiguous and
- * allocated memory block should be passed
- * @retval true/false
- */
- bool getDataFromBuffer(float *out, float *label);
-
- /**
- * @brief set number of class
- * @param[in] number of class
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setClassNum(unsigned int n);
-
- /**
- * @brief set batch size
- * @param[in] n batch size
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setBatchSize(unsigned int n);
-
- /**
- * @brief set feature size
- * @param[in] feature batch size. It is equal to input layer's hidden size
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setFeatureSize(const TensorDim &indim);
-
- /**
* @brief Display Progress
* @param[in] count calculated set ( batch_size size )
* @param[in] type buffer type ( DATA_TRAIN, DATA_VAL, DATA_TEST )
void displayProgress(const int count, float loss);
/**
- * @brief return validation of data set
- * @retval validation
- */
- bool isValid() { return initialized; }
-
- /**
* @brief set property
* @param[in] values values of property
* @retval #ML_ERROR_NONE Successful.
/**
* @brief set property to allow setting user_data for cb
+ * @todo deprecate
* @param[in] values values of property
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
int setProperty(std::vector<void *> values);
- /**
- * @brief set function pointer for each type
- * @param[in] call back function pointer
- * @param[in] user_data user_data of the callback
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setGeneratorFunc(datagen_cb func, void *user_data = nullptr);
-
- /**
- * @brief set train data file name
- * @param[in] path file path
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- virtual int setDataFile(const std::string &path);
-
- /**
- * @brief property type of databuffer
- *
- */
- enum class PropertyType {
- train_data = 0,
- val_data = 1,
- test_data = 2,
- buffer_size = 3,
- unknown = 4
- };
-
protected:
- /**
- * @brief state of the data buffer while getting the data
- */
- enum class DataStatus {
- DATA_NOT_READY = 0,
- DATA_READY = 1,
- DATA_END = 2,
- DATA_ERROR = 3,
- };
-
- /**
- * @brief Set the State And Notify to the condition variable waiting for it
- *
- * @param status status to change
- */
- void setStateAndNotify(const DataStatus status);
-
- DatasetType data_buffer_type; /**< data buffer type */
-
- /** data queue and producer/consumer status related variables */
- std::thread
- batch_producer; /** thread generates a single batches to the queue */
- std::mutex data_lock; /**< data queue mutex */
- std::mutex status_mutex; /**< data status mutex */
- std::condition_variable cv_status; /**< condition variable for status */
- std::vector<std::vector<float>> data_q;
- std::vector<std::vector<float>> label_q;
- DataStatus queue_status; /**< status of the queue */
- unsigned int buf_size; /**< queue size */
- unsigned int cur_bufsize; /**< number of data in the data queue */
-
- TensorDim input_dim; /**< feature size */
- unsigned int class_num; /**< number of class */
- unsigned int batch_size; /**< batch size */
-
- /**< @todo below variable should be owned by databuffer that has fixed size */
- unsigned int samples_per_epoch; /**< size of samples in the dataset */
- unsigned int remaining_samples_per_epoch; /**< size of samples remaining in
- current epoch */
-
- bool is_running; /**< flag to check if running */
- bool initialized; /**< check if current buffer is in valid state */
-
- /**
- * @brief return random int value between min to max
- * @param[in] min minimum vaule
- * @param[in] max maximum value
- * @retval int return value
- */
- int rangeRandom(int min, int max);
-
- std::mt19937 rng;
-
- std::exception_ptr consumer_exception_ptr; /**< exception ptr for consumer to
- catch when producer is dead */
-
- /******************* v2 members ********************/
std::shared_ptr<DataProducer> producer;
std::weak_ptr<BatchQueue> bq_view;
using Props = std::tuple<PropsBufferSize>;
std::unique_ptr<Props> db_props;
+ /// @todo this must be handled from the capi side. favorably, deprecate
+ /// "user_data", callback
/** The user_data to be used for the data generator callback */
void *user_data;
};
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.cpp
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <assert.h>
-#include <climits>
-#include <condition_variable>
-#include <cstring>
-#include <databuffer_file.h>
-#include <functional>
-#include <iomanip>
-#include <mutex>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <sstream>
-#include <stdexcept>
-#include <stdio.h>
-#include <stdlib.h>
-#include <thread>
-
-namespace nntrainer {
-
-static long getFileSize(std::string file_name) {
- std::ifstream file_stream(file_name.c_str(), std::ios::in | std::ios::binary);
- if (file_stream.good()) {
- file_stream.seekg(0, std::ios::end);
- return file_stream.tellg();
- } else {
- return 0;
- }
-}
-
-int DataBufferFromDataFile::init() {
- int status = ML_ERROR_NONE;
-
- status = DataBuffer::init();
- if (status != ML_ERROR_NONE)
- return status;
-
- if (initialized && samples_per_epoch < batch_size) {
- samples_per_epoch = batch_size;
- }
-
- this->remaining_samples_per_epoch = samples_per_epoch;
-
- if (initialized && samples_per_epoch < buf_size) {
- ml_logw("Warning: Total number of train is less than train buffer size. "
- "Train buffer size is set as total number of train");
- buf_size = batch_size;
- }
-
- return ML_ERROR_NONE;
-}
-
-void DataBufferFromDataFile::updateData() {
- std::ifstream file(file_name, std::ios::in | std::ios::binary);
- /// @todo check if file is good
-
- setStateAndNotify(DataStatus::DATA_NOT_READY);
-
- unsigned int I;
- std::vector<unsigned int> mark;
- mark.resize(samples_per_epoch);
- file.clear();
- file.seekg(0, std::ios_base::end);
- uint64_t file_length = file.tellg();
-
- for (unsigned int i = 0; i < samples_per_epoch; ++i) {
- mark[i] = i;
- }
-
- while (is_running) {
- if (mark.size() == 0) {
- setStateAndNotify(DataStatus::DATA_END);
- break;
- }
-
- if (buf_size - cur_bufsize > 0 && remaining_samples_per_epoch > 0) {
- std::vector<float> vec;
- std::vector<float> veclabel;
-
- unsigned int id = rangeRandom(0, mark.size() - 1);
- I = mark[id];
-
- try {
- if (I > samples_per_epoch) {
- throw std::runtime_error(
- "Error: Test case id cannot exceed maximum number of test");
- }
- } catch (...) {
- consumer_exception_ptr = std::current_exception();
- setStateAndNotify(DataStatus::DATA_ERROR);
- return;
- }
-
- mark.erase(mark.begin() + id);
-
- uint64_t position =
- (uint64_t)((I * input_dim.getFeatureLen() + (uint64_t)I * class_num) *
- sizeof(float));
- try {
- if (position > file_length) {
- throw std::runtime_error("Error: Cannot exceed max file size");
- }
- } catch (...) {
- consumer_exception_ptr = std::current_exception();
- setStateAndNotify(DataStatus::DATA_ERROR);
- return;
- }
-
- file.seekg(position, std::ios::beg);
-
- for (unsigned int j = 0; j < input_dim.getFeatureLen(); ++j) {
- float d;
- file.read((char *)&d, sizeof(float));
- vec.push_back(d);
- }
-
- for (unsigned int j = 0; j < class_num; ++j) {
- float d;
- file.read((char *)&d, sizeof(float));
- veclabel.push_back(d);
- }
-
- data_lock.lock();
- data_q.push_back(vec);
- label_q.push_back(veclabel);
- remaining_samples_per_epoch--;
- cur_bufsize++;
- data_lock.unlock();
- }
- if (buf_size == cur_bufsize) {
- setStateAndNotify(DataStatus::DATA_READY);
- }
- }
-
- file.close();
-}
-
-int DataBufferFromDataFile::setDataFile(const std::string &path) {
- int status = ML_ERROR_NONE;
- std::ifstream data_file(path.c_str());
-
- initialized = true;
- if (!data_file.good()) {
- ml_loge("Error: Cannot open data file, Datafile is necessary for training");
- initialized = false;
- return ML_ERROR_INVALID_PARAMETER;
- }
- file_name = path;
-
- ml_logd("datafile has set. path: %s", path.c_str());
-
- return status;
-}
-
-int DataBufferFromDataFile::setFeatureSize(const TensorDim &tdim) {
- int status = ML_ERROR_NONE;
- long file_size = 0;
-
- status = DataBuffer::setFeatureSize(tdim);
- if (status != ML_ERROR_NONE)
- return status;
-
- if (initialized) {
- file_size = getFileSize(file_name);
- samples_per_epoch = static_cast<unsigned int>(
- file_size /
- (class_num * sizeof(int) + input_dim.getFeatureLen() * sizeof(float)));
- if (samples_per_epoch < batch_size) {
- ml_logw("Warning: number of training data is smaller than batch size");
- }
- } else {
- samples_per_epoch = 0;
- }
-
- return status;
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.h
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __DATABUFFER_FILE_H__
-#define __DATABUFFER_FILE_H__
-#ifdef __cplusplus
-
-#include <fstream>
-#include <memory>
-#include <thread>
-#include <vector>
-
-#include <databuffer.h>
-#include <nntrainer_error.h>
-
-namespace nntrainer {
-
-/**
- * @class DataBufferFromDataFile Data Buffer from Raw Data File
- * @brief Data Buffer from reading raw data
- */
-class DataBufferFromDataFile final : public DataBuffer {
-
-public:
- /**
- * @brief Constructor
- */
- DataBufferFromDataFile() : DataBuffer(DatasetType::FILE) {}
-
- /**
- * @brief Constructor
- */
- DataBufferFromDataFile(const std::string &path) : DataBufferFromDataFile() {
- NNTR_THROW_IF(setDataFile(path) != ML_ERROR_NONE, std::invalid_argument)
- << "invalid train file, path: " << path;
- }
-
- /**
- * @brief Destructor
- */
- ~DataBufferFromDataFile() = default;
-
- /**
- * @brief Initialize Buffer with data buffer private variables
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int init();
-
- /**
- * @brief Update Data Buffer ( it is for child thread )
- * @retval void
- */
- void updateData() override;
-
- /**
- * @brief set train data file name
- * @param[in] path file path
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setDataFile(const std::string &path) override;
-
- /**
- * @brief set feature size
- * @param[in] Input Tensor Dimension.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setFeatureSize(const TensorDim &indim) override;
-
-private:
- /**
- * @brief raw data file names
- */
- std::string file_name;
-};
-
-} // namespace nntrainer
-#endif /* __cplusplus */
-#endif /* __DATABUFFER_FILE_H__ */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.cpp
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#include <assert.h>
-#include <climits>
-#include <condition_variable>
-#include <cstring>
-#include <databuffer_func.h>
-#include <functional>
-#include <iomanip>
-#include <mutex>
-#include <nntrainer_error.h>
-#include <nntrainer_log.h>
-#include <sstream>
-#include <stdexcept>
-#include <stdio.h>
-#include <stdlib.h>
-#include <thread>
-
-namespace nntrainer {
-
-int DataBufferFromCallback::init() {
- int status = ML_ERROR_NONE;
-
- status = DataBuffer::init();
- if (status != ML_ERROR_NONE)
- return status;
-
- if (callback == nullptr)
- return ML_ERROR_BAD_ADDRESS;
-
- samples_per_epoch = 0;
-
- if (buf_size > batch_size || buf_size == 0) {
- buf_size = batch_size;
- }
-
- return ML_ERROR_NONE;
-}
-
-int DataBufferFromCallback::setGeneratorFunc(datagen_cb func, void *user_data) {
- if (!func)
- return ML_ERROR_INVALID_PARAMETER;
- callback = func;
- this->user_data = user_data;
- initialized = true;
- return ML_ERROR_NONE;
-}
-
-void DataBufferFromCallback::updateData() {
- int status = ML_ERROR_NONE;
-
- setStateAndNotify(DataStatus::DATA_NOT_READY);
- bool endflag = false;
-
- float **vec_arr = (float **)malloc(sizeof(float *) * 1);
- float **veclabel_arr = (float **)malloc(sizeof(float *) * 1);
-
- float *vec =
- (float *)malloc(sizeof(float) * input_dim.batch() * input_dim.channel() *
- input_dim.height() * input_dim.width());
- float *veclabel =
- (float *)malloc(sizeof(float) * input_dim.batch() * class_num);
-
- try {
- if (vec_arr == nullptr || veclabel_arr == nullptr || vec == nullptr ||
- veclabel == nullptr) {
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
- throw std::runtime_error("Error: assigning error");
- }
- } catch (...) {
- consumer_exception_ptr = std::current_exception();
- setStateAndNotify(DataStatus::DATA_ERROR);
- return;
- }
-
- vec_arr[0] = vec;
- veclabel_arr[0] = veclabel;
-
- while (is_running) {
- endflag = false;
- setStateAndNotify(DataStatus::DATA_NOT_READY);
- if (buf_size - cur_bufsize > 0) {
- /** @todo Update to support multiple inputs later */
- status = callback(vec_arr, veclabel_arr, &endflag, user_data);
- if (endflag) {
- setStateAndNotify(DataStatus::DATA_END);
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
- return;
- }
- if (status != ML_ERROR_NONE) {
- setStateAndNotify(DataStatus::DATA_ERROR);
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
- return;
- }
-
- for (unsigned int i = 0; i < input_dim.batch(); ++i) {
- std::vector<float> v;
- std::vector<float> vl;
- unsigned int I =
- i * input_dim.channel() * input_dim.height() * input_dim.width();
- for (unsigned int j = 0; j < input_dim.channel(); ++j) {
- unsigned int J = j * input_dim.height() * input_dim.width();
- for (unsigned int k = 0; k < input_dim.height() * input_dim.width();
- ++k) {
- unsigned int K = I + J + k;
- v.push_back(vec[K]);
- }
- }
-
- I = i * class_num;
- for (unsigned int j = 0; j < class_num; ++j) {
- vl.push_back(veclabel[I + j]);
- }
-
- data_lock.lock();
- data_q.push_back(v);
- label_q.push_back(vl);
- cur_bufsize++;
- data_lock.unlock();
- }
- }
- if (buf_size == cur_bufsize) {
- setStateAndNotify(DataStatus::DATA_READY);
- }
- }
-
- free(vec);
- free(veclabel);
- free(vec_arr);
- free(veclabel_arr);
-}
-
-} /* namespace nntrainer */
+++ /dev/null
-/**
- * Copyright (C) 2019 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- *
- * @file databuffer_file.h
- * @date 27 April 2020
- * @brief This is buffer object take data from raw files
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs except for NYI items
- *
- */
-
-#ifndef __DATABUFFER_FUNC_H__
-#define __DATABUFFER_FUNC_H__
-#ifdef __cplusplus
-
-#include <functional>
-#include <memory>
-#include <thread>
-#include <vector>
-
-#include <databuffer.h>
-#include <nntrainer-api-common.h>
-
-namespace nntrainer {
-
-/**
- * @class DataBufferFromCallback Data Buffer from callback given by user
- * @brief Data Buffer from callback function
- */
-class DataBufferFromCallback : public DataBuffer {
-public:
- /**
- * @brief Constructor
- */
- DataBufferFromCallback() : DataBuffer(DatasetType::GENERATOR) {}
-
- /**
- * @brief Construct a new Data Buffer From Callback object
- *
- */
- DataBufferFromCallback(datagen_cb func, void *user_data = nullptr) :
- DataBuffer(DatasetType::GENERATOR) {
- setGeneratorFunc(func, user_data);
- }
-
- /**
- * @brief Destructor
- */
- ~DataBufferFromCallback() = default;
-
- /**
- * @brief Initialize Buffer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int init();
-
- /**
- * @brief set function pointer for each type
- * @param[in] call back function pointer
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int setGeneratorFunc(datagen_cb func, void *user_data = nullptr) override;
-
- /**
- * @brief Update Data Buffer ( it is for child thread )
- */
- void updateData() override;
-
-private:
- /**
- *
- * @brief Callback function to get user specific data
- * @param[in] X data 3D float vector type
- * @param[in] Y label 3D float vector type
- * @param[out] status status for error handle
- * @retval true / false generate all data for this epoch
- *
- */
- datagen_cb callback;
-};
-} // namespace nntrainer
-#endif /* __cplusplus */
-#endif /* __DATABUFFER_FUNC_H__ */
'batch_queue.cpp',
'databuffer.cpp',
'databuffer_factory.cpp',
- 'databuffer_file.cpp',
- 'databuffer_func.cpp',
'random_data_producers.cpp',
'func_data_producer.cpp',
'raw_file_data_producer.cpp'
*
* @param v value to set
*/
- void set(const std::string &v);
+ void set(const std::string &v) override;
/**
* @brief return file size
#include <adam.h>
#include <databuffer_factory.h>
-#include <databuffer_file.h>
-#include <databuffer_func.h>
#include <ini_interpreter.h>
#include <model_loader.h>
#include <neuralnet.h>
NeuralNetwork::~NeuralNetwork() {
manager.reset();
model_graph.reset();
-
- std::for_each(data_buffers.begin(), data_buffers.end(), [](auto &buffers) {
- if (buffers) {
- buffers->clear();
- }
- });
}
void NeuralNetwork::setLabels(sharedConstTensors label) {
model_graph.setBatchSize(batch);
manager->setBatchSize(batch);
-
- for (auto &db : data_buffers) {
- if (db != nullptr) {
- int status = db->setBatchSize(batch_size);
- if (status != ML_ERROR_NONE) {
- ml_loge("[model] setting batchsize from data buffer failed");
- throw_status(status);
- }
- }
- }
}
bool NeuralNetwork::validateInput(sharedConstTensors X) {
status = allocate(true);
NN_RETURN_STATUS();
- auto initiate_data_buffer = [this](std::shared_ptr<DataBuffer> &db) {
- /** @todo pass dedicated dimensions for inputs and labels */
- int status = db->setClassNum(getOutputDimension()[0].width());
- NN_RETURN_STATUS();
-
- status = db->setFeatureSize(getInputDimension()[0]);
- NN_RETURN_STATUS();
-
- status = db->init();
- NN_RETURN_STATUS();
-
- return status;
- };
-
- for (auto &db : data_buffers) {
- if (db != nullptr) {
- status = initiate_data_buffer(db);
- }
- NN_RETURN_STATUS();
- }
-
status = train_run();
+ NN_RETURN_STATUS();
/**
* Free the memory needed for training before exiting.
#include <array>
#include <cstring>
-#include <databuffer.h>
#include <iostream>
#include <sstream>
#include <string>
return (unsigned int)NeuralNetwork::PropertyType::unknown;
}
-unsigned int parseDataProperty(std::string property) {
- unsigned int i;
-
- /**
- * @brief Data Properties
- * train_data = 0,
- * val_data = 1,
- * test_data = 2,
- * buffer_size = 3
- */
- std::array<std::string, 5> property_string = {"train_data", "val_data",
- "test_data", "buffer_size"};
-
- for (i = 0; i < property_string.size(); i++) {
- unsigned int size = (property_string[i].size() > property.size())
- ? property_string[i].size()
- : property.size();
-
- if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
- return (i);
- }
- }
-
- return (unsigned int)DataBuffer::PropertyType::unknown;
-}
-
int setUint(unsigned int &val, const std::string &str) {
int status = ML_ERROR_NONE;
try {
'unittest_nntrainer_lazy_tensor',
'unittest_nntrainer_tensor',
'unittest_util_func',
- 'unittest_databuffer_file',
'unittest_nntrainer_modelfile',
'unittest_nntrainer_models',
'unittest_nntrainer_graph',
+++ /dev/null
-/**
- * Copyright (C) 2020 Samsung Electronics Co., Ltd. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * @file unittest_nntrainer_databuffer_file.cpp
- * @date 10 April 2020
- * @brief Unit test databuffer from file.
- * @see https://github.com/nnstreamer/nntrainer
- * @author Jijoong Moon <jijoong.moon@samsung.com>
- * @bug No known bugs
- */
-#include <gtest/gtest.h>
-
-#include "nntrainer_test_util.h"
-#include <databuffer_file.h>
-#include <fstream>
-#include <nntrainer_error.h>
-
-static const std::string getTestResPath(const std::string &file) {
- return getResPath(file, {"test"});
-}
-
-/**
- * @brief Data Buffer
- */
-TEST(nntrainer_DataBuffer, setFeatureSize_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- nntrainer::TensorDim dim;
- dim.setTensorDim("32:1:1:62720");
- status = data_buffer.setClassNum(10);
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setDataFile(getTestResPath("trainingSet.dat"));
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setFeatureSize(dim);
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data Buffer
- */
-TEST(nntrainer_DataBuffer, setBatchSize_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setBatchSize(32);
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data Buffer
- */
-TEST(nntrainer_DataBuffer, setBatchSize_02_n) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setBatchSize(0);
- EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
-}
-
-/**
- * @brief Data Buffer
- */
-TEST(nntrainer_DataBuffer, init_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- nntrainer::TensorDim dim;
- dim.setTensorDim("32:1:1:62720");
- status = data_buffer.setBatchSize(32);
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setClassNum(10);
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setDataFile(getTestResPath("trainingSet.dat"));
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setFeatureSize(dim);
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.init();
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data Buffer set number of Class
- */
-TEST(nntrainer_DataBuffer, setClassNum_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setClassNum(3);
- EXPECT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setClassNum(0);
- EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
-}
-
-/**
- * @brief Data Buffer set number of Class
- */
-TEST(nntrainer_DataBuffer, setClassNum_02_n) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setClassNum(0);
- EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
-}
-
-/**
- * @brief Data Buffer set train Data file
- */
-TEST(nntrainer_DataBuffer, setDataFile_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setDataFile(getTestResPath("trainingSet.dat"));
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data Buffer set train Data file
- */
-TEST(nntrainer_DataBuffer, setDataFile_02_n) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setDataFile("./no_exist.dat");
- EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
-}
-
-/**
- * @brief Data buffer clear all
- */
-TEST(nntrainer_DataBuffer, clear_01_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- nntrainer::TensorDim dim;
- dim.setTensorDim("32:1:1:62720");
- status = data_buffer.setBatchSize(32);
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setClassNum(10);
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setDataFile(getTestResPath("trainingSet.dat"));
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.setFeatureSize(dim);
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.init();
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.run();
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.clear();
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data buffer partial clear
- */
-TEST(nntrainer_DataBuffer, clear_02_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setDataFile(getTestResPath("testSet.dat"));
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.clear();
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data buffer all clear after partial clear
- */
-TEST(nntrainer_DataBuffer, clear_03_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setDataFile(getTestResPath("testSet.dat"));
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.clear();
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Data buffer partial clear after all clear
- */
-TEST(nntrainer_DataBuffer, clear_04_p) {
- int status = ML_ERROR_NONE;
- nntrainer::DataBufferFromDataFile data_buffer;
- status = data_buffer.setDataFile(getTestResPath("testSet.dat"));
- ASSERT_EQ(status, ML_ERROR_NONE);
- status = data_buffer.clear();
- EXPECT_EQ(status, ML_ERROR_NONE);
-}
-
-/**
- * @brief Main gtest
- */
-int main(int argc, char **argv) {
- int result = -1;
-
- try {
- testing::InitGoogleTest(&argc, argv);
- } catch (...) {
- ml_loge("Failed to init gtest\n");
- }
-
- try {
- result = RUN_ALL_TESTS();
- } catch (...) {
- ml_loge("Failed to run test.\n");
- }
-
- return result;
-}
#include <fstream>
-#include <app_context.h>
-#include <databuffer_file.h>
-#include <databuffer_func.h>
#include <neuralnet.h>
#include <nntrainer_error.h>
#include <optimizer.h>
projects / platform / core / ml / nntrainer.git / commitdiff