ifndef NNTRAINER_ROOT
NNTRAINER_ROOT := $(LOCAL_PATH)/../../../libs/arm64-v8a
-NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include
+NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include \
+ $(LOCAL_PATH)/../../../api/capi/include \
+ $(LOCAL_PATH)/../../../api/capi/include/platform
endif
NNTRAINER_APPLICATION := $(LOCAL_PATH)/../..
include $(CLEAR_VARS)
LOCAL_ARM_NEON := true
-LOCAL_CFLAGS += -std=c++14 -Ofast -mcpu=cortex-a53 -Ilz4-nougat/lib
+LOCAL_CFLAGS += -std=c++14 -Ofast -mcpu=cortex-a53 -Ilz4-nougat/lib
LOCAL_LDFLAGS += -Llz4-nougat/lib/obj/local/arm64-v8a/
LOCAL_CXXFLAGS += -std=c++14
LOCAL_CFLAGS += -pthread -fopenmp -fexceptions
#include <stdlib.h>
#include <time.h>
+#include "databuffer.h"
+#include "databuffer_func.h"
#include "neuralnet.h"
+#include "nntrainer_error.h"
#include "tensor.h"
#define TRAINING true
* @brief get data which size is mini batch for train
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_train(float *outVec, float *outLabel, int *status) {
+int getMiniBatch_train(float **outVec, float **outLabel, bool *last) {
std::vector<int> memI;
std::vector<int> memJ;
unsigned int count = 0;
for (unsigned int i = 0; i < total_label_size * data_size; ++i) {
duplicate[i] = false;
}
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
getData(F, o, l, memI[i]);
for (unsigned int j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (unsigned int j = 0; j < total_label_size; ++j)
- outLabel[i * total_label_size + j] = l[j];
+ outLabel[0][i * total_label_size + j] = l[j];
}
F.close();
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
* @brief get data which size is mini batch for validation
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false false : end of data
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_val(float *outVec, float *outLabel, int *status) {
+int getMiniBatch_val(float **outVec, float **outLabel, bool *last) {
std::vector<int> memI;
std::vector<int> memJ;
for (unsigned int i = 0; i < total_label_size * data_size; ++i) {
valduplicate[i] = false;
}
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
getData(F, o, l, memI[i]);
for (unsigned int j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (unsigned int j = 0; j < total_label_size; ++j)
- outLabel[i * total_label_size + j] = l[j];
+ outLabel[0][i * total_label_size + j] = l[j];
}
F.close();
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
}
/**
+ * @brief Data buffer Create & Initialization
+ */
+ std::shared_ptr<nntrainer::DataBufferFromCallback> DB =
+ std::make_shared<nntrainer::DataBufferFromCallback>();
+ DB->setFunc(nntrainer::BUF_TRAIN, getMiniBatch_train);
+ DB->setFunc(nntrainer::BUF_VAL, getMiniBatch_val);
+
+ /**
* @brief Neural Network Create & Initialization
*/
nntrainer::NeuralNetwork NN;
NN.loadFromConfig();
NN.init();
NN.readModel();
+ NN.setDataBuffer((DB));
/**
* @brief Neural Network Train & validation
*/
- NN.train(getMiniBatch_train, getMiniBatch_val, nullptr);
+ NN.train();
/**
* @brief Finalize NN
int status = ML_ERROR_NONE;
- /* handlers for model, layers & optimizer */
+ /* handlers for model, layers, optimizer and dataset */
ml_train_model_h model;
ml_train_layer_h layers[2];
ml_train_optimizer_h optimizer;
+ ml_train_dataset_h dataset;
/* model create */
status = ml_train_model_construct(&model);
status = ml_train_model_compile(model, "loss=cross", NULL);
NN_RETURN_STATUS();
+ /* create dataset */
+ status = ml_train_dataset_create_with_file(&dataset, "trainingSet.dat",
+ "valSet.dat", NULL);
+ NN_RETURN_STATUS();
+
+ /* set property for dataset */
+ status = ml_train_dataset_set_property(dataset, "label_data=label.dat",
+ "buffer_size=100", NULL);
+ NN_RETURN_STATUS();
+
+ /* set dataset */
+ status = ml_train_model_set_dataset(model, dataset);
+ NN_RETURN_STATUS();
+
/* train model with data files : epochs = 10 and store model file named
* "model.bin" */
- status = ml_nnmodel_train_with_file(
- model, "epochs=10", "batch_size=32", "train_data=trainingSet.dat",
- "val_data=trainingSet.dat", "label_data=label.dat", "buffer_size=100",
- "model_file=model.bin", NULL);
+ status = ml_train_model_run(model, "epochs=10", "batch_size=32",
+ "model_file=model.bin", NULL);
NN_RETURN_STATUS();
/* delete model */
static bool alloc_train = false;
static bool alloc_val = false;
-bool gen_data_train(float *outVec, float *outLabel, int *status);
-bool gen_data_val(float *outVec, float *outLabel, int *status);
+int gen_data_train(float **outVec, float **outLabel, bool *last);
+int gen_data_val(float **outVec, float **outLabel, bool *last);
bool file_exists(const char *filename);
bool file_exists(const char *filename) {
* @param[out] status for error handling
* @retval true/false
*/
-bool gen_data_train(float *outVec, float *outLabel, int *status) {
+int gen_data_train(float **outVec, float **outLabel, bool *last) {
int memI[mini_batch];
long file_size;
unsigned int count = 0;
if (!file_exists(file_name)) {
printf("%s does not exists\n", file_name);
- return false;
+ return ML_ERROR_INVALID_PARAMETER;
}
file = fopen(file_name, "r");
}
free(duplicate);
alloc_train = false;
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
get_data(file_name, o, l, memI[i], file_size);
for (j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (j = 0; j < num_class; ++j)
- outLabel[i * num_class + j] = l[j];
+ outLabel[0][i * num_class + j] = l[j];
}
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
* @brief get data which size is mini batch for validation
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false false : end of data
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool gen_data_val(float *outVec, float *outLabel, int *status) {
+int gen_data_val(float **outVec, float **outLabel, bool *last) {
int memI[mini_batch];
unsigned int i, j;
if (count < mini_batch) {
free(valduplicate);
alloc_val = false;
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
get_data(file_name, o, l, memI[i], file_size);
for (j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (j = 0; j < num_class; ++j)
- outLabel[i * num_class + j] = l[j];
+ outLabel[0][i * num_class + j] = l[j];
}
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
int main(int argc, char *argv[]) {
int status = ML_ERROR_NONE;
- /* handlers for model, layers & optimizer */
+ /* handlers for model, layers, optimizer and dataset */
ml_train_model_h model;
ml_train_layer_h layers[2];
ml_train_optimizer_h optimizer;
+ ml_train_dataset_h dataset;
/* model create */
status = ml_train_model_construct(&model);
status = ml_train_model_compile(model, "loss=cross", NULL);
NN_RETURN_STATUS();
+ /* create dataset */
+ status = ml_train_dataset_create_with_generator(&dataset, gen_data_train,
+ gen_data_val, NULL);
+ NN_RETURN_STATUS();
+
+ /* set property for dataset */
+ status = ml_train_dataset_set_property(dataset, "buffer_size=32", NULL);
+ NN_RETURN_STATUS();
+
+ /* set dataset */
+ status = ml_train_model_set_dataset(model, dataset);
+ NN_RETURN_STATUS();
+
/* train model with data files : epochs = 10 and store model file named
* "model.bin" */
- status = ml_nnmodel_train_with_generator(
- model, gen_data_train, gen_data_val, NULL, "epochs=10", "batch_size=32",
- "model_file=model.bin", "buffer_size = 32", NULL);
+ status = ml_train_model_run(model, "epochs=10", "batch_size=32",
+ "model_file=model.bin", NULL);
NN_RETURN_STATUS();
/* delete model */
status = ml_train_model_compile(handle, NULL);
if (status != ML_ERROR_NONE)
return status;
- status = ml_nnmodel_train_with_file(handle, NULL);
+ status = ml_train_model_run(handle, NULL);
if (status != ML_ERROR_NONE)
return status;
status = ml_train_model_destroy(handle);
LOCAL_SHARED_LIBRARIES := nntrainer
-LOCAL_STATIC_LIBRARIES := tensorflow-lite
+LOCAL_STATIC_LIBRARIES := tensorflow-lite
LOCAL_C_INCLUDES += $(TFLITE_INCLUDES) $(NNTRAINER_INCLUDES)
ifndef NNTRAINER_ROOT
NNTRAINER_ROOT := $(LOCAL_PATH)/../../../libs/arm64-v8a
-NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include
+NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include \
+ $(LOCAL_PATH)/../../../api/capi/include \
+ $(LOCAL_PATH)/../../../api/capi/include/platform
endif
include $(CLEAR_VARS)
#include <stdlib.h>
#include <time.h>
+#include "databuffer.h"
+#include "databuffer_func.h"
#include "neuralnet.h"
+#include "nntrainer_error.h"
#include "tensor.h"
#define TRAINING true
* @brief get data which size is mini batch for train
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_train(float *outVec, float *outLabel, int *status) {
+int getMiniBatch_train(float **outVec, float **outLabel, bool *last) {
std::vector<int> memI;
std::vector<int> memJ;
unsigned int count = 0;
for (unsigned int i = 0; i < total_label_size * data_size; ++i) {
duplicate[i] = false;
}
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
getData(F, o, l, memI[i]);
for (unsigned int j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (unsigned int j = 0; j < total_label_size; ++j)
- outLabel[i * total_label_size + j] = l[j];
+ outLabel[0][i * total_label_size + j] = l[j];
}
F.close();
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
* @brief get data which size is mini batch for validation
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false false : end of data
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_val(float *outVec, float *outLabel, int *status) {
+int getMiniBatch_val(float **outVec, float **outLabel, bool *last) {
std::vector<int> memI;
std::vector<int> memJ;
for (unsigned int i = 0; i < total_label_size * data_size; ++i) {
valduplicate[i] = false;
}
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
getData(F, o, l, memI[i]);
for (unsigned int j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (unsigned int j = 0; j < total_label_size; ++j)
- outLabel[i * total_label_size + j] = l[j];
+ outLabel[0][i * total_label_size + j] = l[j];
}
F.close();
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
}
/**
+ * @brief Data buffer Create & Initialization
+ */
+ std::shared_ptr<nntrainer::DataBufferFromCallback> DB =
+ std::make_shared<nntrainer::DataBufferFromCallback>();
+ DB->setFunc(nntrainer::BUF_TRAIN, getMiniBatch_train);
+ DB->setFunc(nntrainer::BUF_VAL, getMiniBatch_val);
+
+ /**
* @brief Neural Network Create & Initialization
*/
- nntrainer::NeuralNetwork NN(config);
+ nntrainer::NeuralNetwork NN;
+ NN.setConfig(config);
NN.loadFromConfig();
NN.init();
NN.readModel();
+ NN.setDataBuffer((DB));
/**
* @brief Neural Network Train & validation
*/
- NN.train(getMiniBatch_train, getMiniBatch_val, nullptr);
+ NN.train();
/**
* @brief Finalize NN
typedef void *ml_train_optimizer_h;
/**
+ * @brief A handle of an NNTrainer dataset.
+ * @since_tizen 6.x
+ */
+typedef void *ml_train_dataset_h;
+
+/**
* @brief Enumeration for the neural network layer type of NNTrainer.
* @since_tizen 6.x
*/
} ml_train_summary_type_e;
/**
+ * @brief Dataset generator callback function for train/valid/test data.
+ * @details The Containers passed will already be allocated with sufficient
+ * space to contain the data by the caller. This function should return a batch
+ * of input and label of data in the passed containers. The callback should fill
+ * the data row-wise in the containers obliging to the input shape set for the
+ * model. The order of the inputs in case of multiple input layers will be
+ * determined based on the sequence of addition of the input layers to the
+ * model.
+ * @note This function can be called multiple times in parallel.
+ * @param[out] input Container to hold all the input data.
+ * @param[out] label Container to hold corresponding label data.
+ * @param[out] last Container to notify if data is finished. Set true if no more
+ * data to provide, else set false.
+ * @return @c 0 on success. Otherwise a negative error value.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
+ */
+typedef int (*ml_train_datagen_cb)(float **input, float **label, bool *last);
+
+/**
* @brief Constructs the neural network model.
* @details Use this function to create Neural Netowrk Model.
* @since_tizen 6.x
int ml_train_model_compile(ml_train_model_h model, ...);
/**
- * @brief train the neural network model.
- * @details Use this function to train neural network model
- * @since_tizen 6.x
- * @param[in] model The NNTrainer model handler from the given description.
- * @param[in] ... hyper parmeter for train model
- * @return @c 0 on success. Otherwise a negative error value.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
- */
-int ml_nnmodel_train_with_file(ml_train_model_h model, ...);
-
-/**
- * @brief train the neural network model.
- * @details Use this function to train neural network model
+ * @brief Train the neural network model.
+ * @details Use this function to train the compiled neural network model with
+ * the passed training hyperparameters. This function will return once the
+ * training along with requested validation and testing is completed.
* @since_tizen 6.x
* @param[in] model The NNTrainer model handler from the given description.
- * @param[in] train_func function pointer for train
- * @param[in] val_func function pointer for val
- * @param[in] test_func function pointer for test
* @param[in] ... hyper parmeter for train model
* @return @c 0 on success. Otherwise a negative error value.
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
*/
-int ml_nnmodel_train_with_generator(ml_train_model_h model,
- bool (*train_func)(float *, float *, int *),
- bool (*val_func)(float *, float *, int *),
- bool (*test_func)(float *, float *, int *),
- ...);
+int ml_train_model_run(ml_train_model_h model, ...);
/**
* @brief Destructs the neural network model.
ml_train_optimizer_h optimizer);
/**
+ * @brief Set the dataset (data provider) for the neural network model.
+ * @details Use this function to set dataset for running the model. The dataset
+ * will provide training, validation and test data for the model. Unsets the
+ * previous dataset if any. This transfers the ownership of the dataset to
+ * the network. No need to delete the dataset once it is set to a model.
+ * @since_tizen 6.x
+ * @param[in] model The NNTrainer model handler.
+ * @param[in] dataset The NNTrainer dataset handler.
+ * @return @c 0 on success. Otherwise a negative error value.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
+ */
+int ml_train_model_set_dataset(ml_train_model_h model,
+ ml_train_dataset_h dataset);
+
+/**
* @brief Create the neural network layer.
* @details Use this function to create Neural Netowrk Layer.
* @since_tizen 6.x
int ml_train_optimizer_set_property(ml_train_optimizer_h optimizer, ...);
/**
+ * @brief Create a dataset with generators to feed to a neural network.
+ * @details Use this function to create a Neural Network Dataset using
+ * generators. The generators will provide data representing a single input
+ * batch. When setting this dataset to a model, the data generated by the
+ * generators should match the input and the label shape for the model.
+ * @since_tizen 6.x
+ * @param[out] dataset The NNTrainer Dataset handler from the given description.
+ * @param[in] train_cb The dataset generator for training.
+ * @param[in] valid_cb The dataset generator for validating. Can be null.
+ * @param[in] test_cb The dataset generator for testing. Can be null.
+ * @return @c 0 on success. Otherwise a negative error value.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
+ */
+int ml_train_dataset_create_with_generator(ml_train_dataset_h *dataset,
+ ml_train_datagen_cb train_cb,
+ ml_train_datagen_cb valid_cb,
+ ml_train_datagen_cb test_cb);
+
+/**
+ * @brief Create a dataset with files to feed to a neural network.
+ * @details Use this function to create a Neural Network Dataset using
+ * files.
+ * @since_tizen 6.x
+ * @param[out] dataset The NNTrainer Dataset handler from the given description.
+ * @param[in] train_fle The dataset file for training.
+ * @param[in] valid_file The dataset file for validating. Can be null.
+ * @param[in] test_file The dataset file for testing. Can be null.
+ * @return @c 0 on success. Otherwise a negative error value.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
+ */
+int ml_train_dataset_create_with_file(ml_train_dataset_h *dataset,
+ const char *train_file,
+ const char *valid_file,
+ const char *test_file);
+
+/**
+ * @brief Destroy the neural network dataset.
+ * @details Use this function to destroy dataset. Fails if dataset is owned by a
+ * model.
+ * @since_tizen 6.x
+ * @param[in] dataset The NNTrainer dataset handler.
+ * @return @c 0 on success. Otherwise a negative error value.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter.
+ */
+int ml_train_dataset_destroy(ml_train_dataset_h dataset);
+
+/**
+ * @brief Set the neural network dataset Property.
+ * @details Use this function to set dataset Property.
+ * @since_tizen 6.x
+ * @param[in] dataset The NNTrainer dataset handler.
+ * @param[in] ... Property values with NULL for termination.
+ * @return @c 0 on success. Otherwise a negative error value.
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER Invalid parameter.
+ */
+int ml_train_dataset_set_property(ml_train_dataset_h dataset, ...);
+
+/**
* @}
*/
#ifdef __cplusplus
*/
/**
* @file nntrainer_internal.h
- * @date 02 April 2020
+ * @date 13 April 2020
* @brief NNTrainer C-API Internal Header.
* This allows to construct and control NNTrainer Model.
* @see https://github.com/nnstreamer/nntrainer
* @author Jijoong Moon <jijoong.moon@samsung.com>
+ * @author Parichay Kapoor <pk.kapoor@samsung.com>
* @bug No known bugs except for NYI items
*/
extern "C" {
#endif /* __cplusplus */
+/**
+ * @brief Struct to wrap neural network layer for the API
+ */
typedef struct {
uint magic;
std::shared_ptr<nntrainer::Layer> layer;
bool in_use;
-} ml_nnlayer;
+} ml_train_layer;
+/**
+ * @brief Struct to wrap neural network optimizer for the API
+ */
typedef struct {
uint magic;
std::shared_ptr<nntrainer::Optimizer> optimizer;
bool in_use;
-} ml_nnopt;
+} ml_train_optimizer;
+/**
+ * @brief Struct to wrap data buffer for the API
+ */
+typedef struct {
+ uint magic;
+ std::shared_ptr<nntrainer::DataBuffer> data_buffer;
+ bool in_use;
+} ml_train_dataset;
+
+/**
+ * @brief Struct to wrap neural network model for the API
+ */
typedef struct {
uint magic;
std::shared_ptr<nntrainer::NeuralNetwork> network;
- std::unordered_map<std::string, ml_nnlayer *> layers_map;
- ml_nnopt *optimizer;
-} ml_nnmodel;
-
-#define ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model) \
- do { \
- if (!model) { \
- ml_loge("Error: Invalid Parameter : model is empty."); \
- return ML_ERROR_INVALID_PARAMETER; \
- } \
- nnmodel = (ml_nnmodel *)model; \
- if (nnmodel->magic != ML_NNTRAINER_MAGIC) { \
- ml_loge("Error: Invalid Parameter : nnmodel is invalid."); \
- return ML_ERROR_INVALID_PARAMETER; \
- } \
- } while (0)
+ std::unordered_map<std::string, ml_train_layer *> layers_map;
+ ml_train_optimizer *optimizer;
+ ml_train_dataset *dataset;
+} ml_train_model;
-#define ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer) \
- do { \
- if (!layer) { \
- ml_loge("Error: Invalid Parameter : layer is empty."); \
- return ML_ERROR_INVALID_PARAMETER; \
- } \
- nnlayer = (ml_nnlayer *)layer; \
- if (nnlayer->magic != ML_NNTRAINER_MAGIC) { \
- ml_loge("Error: Invalid Parameter : nnlayer is invalid."); \
- return ML_ERROR_INVALID_PARAMETER; \
- } \
+/**
+ * @brief Check validity of the user passed arguments
+ */
+#define ML_NNTRAINER_CHECK_VALIDATION(obj, obj_h, obj_type, obj_name) \
+ do { \
+ if (!obj_h) { \
+ ml_loge("Error: Invalid Parameter : %s is empty.", obj_name); \
+ return ML_ERROR_INVALID_PARAMETER; \
+ } \
+ obj = (obj_type *)obj_h; \
+ if (obj->magic != ML_NNTRAINER_MAGIC) { \
+ ml_loge("Error: Invalid Parameter : %s is invalid.", obj_name); \
+ return ML_ERROR_INVALID_PARAMETER; \
+ } \
} while (0)
-#define ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, opt) \
- do { \
- if (!opt) { \
- ml_loge("Error: Invalid Parameter : optimizer is empty."); \
- return ML_ERROR_INVALID_PARAMETER; \
- } \
- nnopt = (ml_nnopt *)opt; \
- if (nnopt->magic != ML_NNTRAINER_MAGIC) { \
- ml_loge("Error: Invalid Parameter : nnoptimizer is invalid."); \
- return ML_ERROR_INVALID_PARAMETER; \
- } \
- } while (0)
+#define ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model) \
+ ML_NNTRAINER_CHECK_VALIDATION(nnmodel, model, ml_train_model, "model")
+
+#define ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer) \
+ ML_NNTRAINER_CHECK_VALIDATION(nnlayer, layer, ml_train_layer, "layer")
+
+#define ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, opt) \
+ ML_NNTRAINER_CHECK_VALIDATION(nnopt, opt, ml_train_optimizer, "optimizer")
+
+#define ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset) \
+ ML_NNTRAINER_CHECK_VALIDATION(nndataset, dataset, ml_train_dataset, "dataset")
/**
* @brief Get neural network layer from the model with the given name.
* @author Jijoong Moon <jijoong.moon@samsung.com>
* @bug No known bugs except for NYI items
*/
+#include <databuffer.h>
+#include <databuffer_file.h>
+#include <databuffer_func.h>
#include <ml-api-common.h>
#include <neuralnet.h>
#include <nntrainer_internal.h>
if (model == NULL)
return ML_ERROR_INVALID_PARAMETER;
- ml_nnmodel *nnmodel = new ml_nnmodel;
+ ml_train_model *nnmodel = new ml_train_model;
nnmodel->magic = ML_NNTRAINER_MAGIC;
nnmodel->optimizer = NULL;
+ nnmodel->dataset = NULL;
*model = nnmodel;
int ml_train_model_construct_with_conf(const char *model_conf,
ml_train_model_h *model) {
int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
+ ml_train_model *nnmodel;
std::shared_ptr<nntrainer::NeuralNetwork> NN;
returnable f;
if (status != ML_ERROR_NONE)
return status;
- nnmodel = (ml_nnmodel *)(*model);
+ nnmodel = (ml_train_model *)(*model);
NN = nnmodel->network;
f = [&]() { return NN->setConfig(model_conf); };
int ml_train_model_compile(ml_train_model_h model, ...) {
int status = ML_ERROR_NONE;
const char *data;
- ml_nnmodel *nnmodel;
+ ml_train_model *nnmodel;
returnable f;
ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
return status;
}
-int ml_nnmodel_train_with_file(ml_train_model_h model, ...) {
+int ml_train_model_run(ml_train_model_h model, ...) {
int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
+ ml_train_model *nnmodel;
const char *data;
std::vector<std::string> arg_list;
return status;
}
-int ml_nnmodel_train_with_generator(ml_train_model_h model,
- bool (*train_func)(float *, float *, int *),
- bool (*val_func)(float *, float *, int *),
- bool (*test_func)(float *, float *, int *),
- ...) {
- int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
- const char *data;
-
- std::vector<std::string> arg_list;
- va_list arguments;
- va_start(arguments, (test_func));
- while ((data = va_arg(arguments, const char *))) {
- arg_list.push_back(data);
- }
-
- va_end(arguments);
-
- ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
- std::shared_ptr<nntrainer::NeuralNetwork> NN;
- NN = nnmodel->network;
-
- returnable f = [&]() {
- return NN->train((train_func), (val_func), (test_func), arg_list);
- };
-
- status = nntrainer_exception_boundary(f);
-
- return status;
-}
-
int ml_train_model_destroy(ml_train_model_h model) {
int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
+ ml_train_model *nnmodel;
ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
if (nnmodel->optimizer)
delete nnmodel->optimizer;
+ if (nnmodel->dataset)
+ delete nnmodel->dataset;
for (auto &x : nnmodel->layers_map)
delete (x.second);
nnmodel->layers_map.clear();
int ml_train_model_add_layer(ml_train_model_h model, ml_train_layer_h layer) {
int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
- ml_nnlayer *nnlayer;
+ ml_train_model *nnmodel;
+ ml_train_layer *nnlayer;
ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer);
+ if (nnlayer->in_use) {
+ ml_loge("Layer already in use.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
std::shared_ptr<nntrainer::NeuralNetwork> NN;
std::shared_ptr<nntrainer::Layer> NL;
int ml_train_model_set_optimizer(ml_train_model_h model,
ml_train_optimizer_h optimizer) {
int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
- ml_nnopt *nnopt;
+ ml_train_model *nnmodel;
+ ml_train_optimizer *nnopt;
ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, optimizer);
+ if (nnopt->in_use) {
+ ml_loge("Optimizer already in use.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
std::shared_ptr<nntrainer::NeuralNetwork> NN;
std::shared_ptr<nntrainer::Optimizer> opt;
return status;
}
+int ml_train_model_set_dataset(ml_train_model_h model,
+ ml_train_dataset_h dataset) {
+ int status = ML_ERROR_NONE;
+ ml_train_model *nnmodel;
+ ml_train_dataset *nndataset;
+
+ ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
+ ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset);
+
+ if (nndataset->in_use) {
+ ml_loge("Dataset already in use.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ std::shared_ptr<nntrainer::NeuralNetwork> NN;
+ std::shared_ptr<nntrainer::DataBuffer> data;
+
+ NN = nnmodel->network;
+ data = nndataset->data_buffer;
+
+ returnable f = [&]() { return NN->setDataBuffer(data); };
+
+ status = nntrainer_exception_boundary(f);
+ if (status == ML_ERROR_NONE) {
+ nndataset->in_use = true;
+ if (nnmodel->dataset)
+ nnmodel->dataset->in_use = false;
+ nnmodel->dataset = nndataset;
+ }
+
+ return status;
+}
+
int ml_train_model_get_layer(ml_train_model_h model, const char *layer_name,
ml_train_layer_h *layer) {
int status = ML_ERROR_NONE;
- ml_nnmodel *nnmodel;
+ ml_train_model *nnmodel;
ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model);
std::shared_ptr<nntrainer::NeuralNetwork> NN;
std::shared_ptr<nntrainer::Layer> NL;
- std::unordered_map<std::string, ml_nnlayer *>::iterator layer_iter =
+ std::unordered_map<std::string, ml_train_layer *>::iterator layer_iter =
nnmodel->layers_map.find(std::string(layer_name));
/** if layer found in layers_map, return layer */
if (layer_iter != nnmodel->layers_map.end()) {
if (status != ML_ERROR_NONE)
return status;
- ml_nnlayer *nnlayer = new ml_nnlayer;
+ ml_train_layer *nnlayer = new ml_train_layer;
nnlayer->magic = ML_NNTRAINER_MAGIC;
nnlayer->layer = NL;
nnlayer->in_use = true;
int ml_train_layer_create(ml_train_layer_h *layer, ml_train_layer_type_e type) {
int status = ML_ERROR_NONE;
returnable f;
- ml_nnlayer *nnlayer = new ml_nnlayer;
+ ml_train_layer *nnlayer = new ml_train_layer;
nnlayer->magic = ML_NNTRAINER_MAGIC;
try {
int ml_train_layer_destroy(ml_train_layer_h layer) {
int status = ML_ERROR_NONE;
- ml_nnlayer *nnlayer;
+ ml_train_layer *nnlayer;
ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer);
int ml_train_layer_set_property(ml_train_layer_h layer, ...) {
int status = ML_ERROR_NONE;
- ml_nnlayer *nnlayer;
+ ml_train_layer *nnlayer;
const char *data;
ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer);
ml_train_optimizer_type_e type) {
int status = ML_ERROR_NONE;
- ml_nnopt *nnopt = new ml_nnopt;
+ ml_train_optimizer *nnopt = new ml_train_optimizer;
nnopt->magic = ML_NNTRAINER_MAGIC;
nnopt->optimizer = std::make_shared<nntrainer::Optimizer>();
nnopt->in_use = false;
int ml_train_optimizer_destroy(ml_train_optimizer_h optimizer) {
int status = ML_ERROR_NONE;
- ml_nnopt *nnopt;
+ ml_train_optimizer *nnopt;
ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, optimizer);
int ml_train_optimizer_set_property(ml_train_optimizer_h optimizer, ...) {
int status = ML_ERROR_NONE;
- ml_nnopt *nnopt;
+ ml_train_optimizer *nnopt;
const char *data;
- nnopt = (ml_nnopt *)optimizer;
+ nnopt = (ml_train_optimizer *)optimizer;
ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, optimizer);
std::vector<std::string> arg_list;
return status;
}
+int ml_train_dataset_create_with_generator(ml_train_dataset_h *dataset,
+ ml_train_datagen_cb train_cb,
+ ml_train_datagen_cb valid_cb,
+ ml_train_datagen_cb test_cb) {
+ int status = ML_ERROR_NONE;
+
+ std::shared_ptr<nntrainer::DataBufferFromCallback> data_buffer =
+ std::make_shared<nntrainer::DataBufferFromCallback>();
+
+ status = data_buffer->setFunc(nntrainer::BUF_TRAIN, train_cb);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+
+ status = data_buffer->setFunc(nntrainer::BUF_VAL, valid_cb);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+
+ status = data_buffer->setFunc(nntrainer::BUF_TEST, test_cb);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+
+ ml_train_dataset *nndataset = new ml_train_dataset;
+ nndataset->magic = ML_NNTRAINER_MAGIC;
+ nndataset->data_buffer = data_buffer;
+ nndataset->in_use = false;
+
+ *dataset = nndataset;
+ return status;
+}
+
+int ml_train_dataset_create_with_file(ml_train_dataset_h *dataset,
+ const char *train_file,
+ const char *valid_file,
+ const char *test_file) {
+ int status = ML_ERROR_NONE;
+
+ std::shared_ptr<nntrainer::DataBufferFromDataFile> data_buffer =
+ std::make_shared<nntrainer::DataBufferFromDataFile>();
+
+ if (train_file) {
+ status = data_buffer->setDataFile(train_file, nntrainer::DATA_TRAIN);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+ } else {
+ ml_loge("Train data file must be valid.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ if (valid_file) {
+ status = data_buffer->setDataFile(valid_file, nntrainer::DATA_VAL);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+ }
+
+ if (test_file) {
+ status = data_buffer->setDataFile(test_file, nntrainer::DATA_TEST);
+ if (status != ML_ERROR_NONE) {
+ return status;
+ }
+ }
+
+ ml_train_dataset *nndataset = new ml_train_dataset;
+ nndataset->magic = ML_NNTRAINER_MAGIC;
+ nndataset->data_buffer = data_buffer;
+ nndataset->in_use = false;
+
+ *dataset = nndataset;
+ return status;
+}
+
+int ml_train_dataset_set_property(ml_train_dataset_h dataset, ...) {
+ int status = ML_ERROR_NONE;
+ ml_train_dataset *nndataset;
+ const char *data;
+
+ ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset);
+
+ std::vector<std::string> arg_list;
+ va_list arguments;
+ va_start(arguments, dataset);
+
+ while ((data = va_arg(arguments, const char *))) {
+ arg_list.push_back(data);
+ }
+
+ va_end(arguments);
+
+ returnable f = [&]() {
+ return nndataset->data_buffer->setProperty(arg_list);
+ };
+ status = nntrainer_exception_boundary(f);
+
+ return status;
+}
+
+int ml_train_dataset_destroy(ml_train_dataset_h dataset) {
+ int status = ML_ERROR_NONE;
+ ml_train_dataset *nndataset;
+
+ ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset);
+
+ if (nndataset->in_use) {
+ ml_loge("Cannot delete dataset already set to a model."
+ "Delete model will delete this dataset.");
+ return ML_ERROR_INVALID_PARAMETER;
+ }
+
+ delete nndataset;
+ return status;
+}
+
#ifdef __cplusplus
}
#endif
$(NNTRAINER_ROOT)/nntrainer/src/flatten_layer.cpp
NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer/include \
- $(NNTRAINER_ROOT)/api/capi/include/platform
+ $(NNTRAINER_ROOT)/api/capi/include \
+ $(NNTRAINER_ROOT)/api/capi/include/platform
INIPARSER_SRCS := $(INIPARSER_ROOT)/src/iniparser.c \
$(INIPARSER_ROOT)/src/dictionary.c
'-Waggregate-return',
'-Wold-style-definition',
'-Wdeclaration-after-statement',
- '-Wno-error=varargs'
+ '-Wno-error=varargs'
]
foreach extra_arg : warning_flags
#include <util_func.h>
#include <vector>
-/*
+/**
* @brief Number of Data Set
*/
#define NBUFTYPE 4
} DataType;
/**
+ * @brief Enumeration for data buffer type
+ * 0. DATA_BUFFER_GENERATOR
+ * 1. DATA_BUFFER_FILE
+ * 2. DATA_BUFFER_UNKNOWN
+ */
+typedef enum {
+ DATA_BUFFER_GENERATOR,
+ DATA_BUFFER_FILE,
+ DATA_BUFFER_UNKNOWN
+} DataBufferType;
+
+/**
* @class DataBuffer Data Buffers
* @brief Data Buffer for read and manage data
*/
* @brief Create Buffer
* @retval DataBuffer
*/
- DataBuffer() :
+ DataBuffer(DataBufferType type) :
train_running(),
val_running(),
test_running(),
train_thread(),
val_thread(),
- test_thread() {
+ test_thread(),
+ data_buffer_type(type) {
SET_VALIDATION(false);
class_num = 0;
cur_train_bufsize = 0;
bool *getValidation() { return validation; }
/**
+ * @brief set property
+ * @param[in] values values of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(std::vector<std::string> values);
+
+ /**
* @brief status of thread
*/
DataStatus trainReadyFlag;
DataStatus valReadyFlag;
DataStatus testReadyFlag;
+ enum class PropertyType {
+ train_data = 0,
+ val_data = 1,
+ test_data = 2,
+ label_data = 3,
+ buffer_size = 4,
+ unknown = 5
+ };
+
protected:
/**
* @brief Data Queues for each data set
int rangeRandom(int min, int max);
std::mt19937 rng;
+
+ /**
+ * @brief The type of data buffer
+ */
+ DataBufferType data_buffer_type;
+
+ /**
+ * @brief set property
+ * @param[in] type type of property
+ * @param[in] value string value of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ virtual int setProperty(const DataBuffer::PropertyType type,
+ std::string &value);
};
} // namespace nntrainer
/**
* @brief Constructor
*/
- DataBufferFromDataFile(){};
+ DataBufferFromDataFile() : DataBuffer(DataBufferType::DATA_BUFFER_FILE){};
/**
* @brief Destructor
*/
int setFeatureSize(TensorDim indim);
+ /**
+ * @brief set property
+ * @param[in] type type of property
+ * @param[in] value string value of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(const PropertyType type, std::string &value);
+
private:
/**
* @brief raw data file names
#include <functional>
#include <iostream>
#include <memory>
+#include <nntrainer.h>
#include <thread>
#include <vector>
namespace nntrainer {
/**
+ * @brief Dataset generator callback type declaration
+ */
+typedef std::function<std::remove_pointer<ml_train_datagen_cb>::type>
+ datagen_cb;
+
+/**
* @class DataBufferFromCallback Data Buffer from callback given by user
* @brief Data Buffer from callback function
*/
/**
* @brief Constructor
*/
- DataBufferFromCallback(){};
+ DataBufferFromCallback() :
+ DataBuffer(DataBufferType::DATA_BUFFER_GENERATOR){};
/**
* @brief Destructor
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int setFunc(BufferType type,
- std::function<bool(float *, float *, int *)> func);
+ int setFunc(BufferType type, datagen_cb func);
/**
* @brief Update Data Buffer ( it is for child thread )
*/
void updateData(BufferType type);
+ /**
+ * @brief set property
+ * @param[in] type type of property
+ * @param[in] value string value of property
+ * @retval #ML_ERROR_NONE Successful.
+ * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
+ */
+ int setProperty(const PropertyType type, std::string &value);
+
private:
/**
*
* @retval true / false generate all data for this epoch
*
*/
- std::function<bool(float *, float *, int *)> callback_train;
- std::function<bool(float *, float *, int *)> callback_val;
- std::function<bool(float *, float *, int *)> callback_test;
+ datagen_cb callback_train;
+ datagen_cb callback_val;
+ datagen_cb callback_test;
};
} // namespace nntrainer
#endif /* __cplusplus */
* mini batch size data per every call.
* @param[in] test_func callback function to get test data. This provides
* mini batch size data per every call.
- * @retval #ML_ERROR_NONE Successful.
- * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
- */
- int train(std::function<bool(float *, float *, int *)> function_train,
- std::function<bool(float *, float *, int *)> function_val,
- std::function<bool(float *, float *, int *)> function_test);
-
- /**
- * @brief Run NeuralNetwork train with callback function by user
- * @param[in] train_func callback function to get train data. This provides
- * mini batch size data per every call.
- * @param[in] val_func callback function to get validation data. This provides
- * mini batch size data per every call.
- * @param[in] test_func callback function to get test data. This provides
- * mini batch size data per every call.
* @param[in] values hyper-parameter list
* @retval #ML_ERROR_NONE Successful.
* @retval #ML_ERROR_INVALID_PARAMETER invalid parameter.
*/
- int train(std::function<bool(float *, float *, int *)> function_train,
- std::function<bool(float *, float *, int *)> function_val,
- std::function<bool(float *, float *, int *)> function_test,
- std::vector<std::string> values);
+ int setDataBuffer(std::shared_ptr<DataBuffer> data_buffer);
/**
* @brief check neural network whether the hyper-parameters are set.
enum class PropertyType {
loss = 0,
cost = 1,
- train_data = 2,
- val_data = 3,
- test_data = 4,
- label_data = 5,
- buffer_size = 6,
- batch_size = 7,
- epochs = 8,
- model_file = 9,
- continue_train = 10,
- unknown = 11,
+ batch_size = 2,
+ epochs = 3,
+ model_file = 4,
+ continue_train = 5,
+ unknown = 6
};
private:
unsigned int parseNetProperty(std::string property);
/**
+ * @brief Parsing Data Buffer Property
+ * @param[in] property string to be parsed
+ * @retval int enumerated type
+ */
+unsigned int parseDataProperty(std::string property);
+
+/**
* @brief check str to be int and assign
* @param[out] val assign variable
* @param[in] str input string
nntrainer_inc = [
- include_directories('./include')
+ include_directories('./include'),
+ include_directories('../api/capi/include')
]
# pc file is not present for 'ml-api-common' yet
int BatchNormalizationLayer::setOptimizer(Optimizer &opt) {
this->opt.setType(opt.getType());
this->opt.setOptParam(opt.getOptParam());
- this->epsilon = epsilon;
return this->opt.initialize(dim, false);
}
#include <mutex>
#include <nntrainer_error.h>
#include <nntrainer_log.h>
+#include <parse_util.h>
#include <sstream>
#include <stdexcept>
#include <stdio.h>
std::cout.flush();
}
+int DataBuffer::setProperty(std::vector<std::string> values) {
+ int status = ML_ERROR_NONE;
+
+ for (unsigned int i = 0; i < values.size(); ++i) {
+ std::string key;
+ std::string value;
+ status = getKeyValue(values[i], key, value);
+ NN_RETURN_STATUS();
+
+ unsigned int type = parseDataProperty(key);
+ if (value.empty())
+ return ML_ERROR_INVALID_PARAMETER;
+
+ status = setProperty(static_cast<PropertyType>(type), value);
+ NN_RETURN_STATUS();
+ }
+
+ return status;
+}
+
+int DataBuffer::setProperty(const PropertyType type, std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ switch (type) {
+ case PropertyType::buffer_size:
+ int size;
+ status = setInt(size, value);
+ NN_RETURN_STATUS();
+ status = this->setBufSize(size);
+ NN_RETURN_STATUS();
+ break;
+ default:
+ ml_loge("Error: Unknown Data Buffer Property Key");
+ status = ML_ERROR_INVALID_PARAMETER;
+ break;
+ }
+
+ return status;
+}
+
} /* namespace nntrainer */
return status;
}
+int DataBufferFromDataFile::setProperty(const PropertyType type,
+ std::string &value) {
+ int status = ML_ERROR_NONE;
+
+ if (data_buffer_type != DATA_BUFFER_FILE)
+ return ML_ERROR_INVALID_PARAMETER;
+
+ switch (type) {
+ case PropertyType::train_data:
+ status = this->setDataFile(value, DATA_TRAIN);
+ break;
+ case PropertyType::val_data:
+ status = this->setDataFile(value, DATA_VAL);
+ break;
+ case PropertyType::test_data:
+ status = this->setDataFile(value, DATA_TEST);
+ break;
+ case PropertyType::label_data:
+ status = this->setDataFile(value, DATA_LABEL);
+ break;
+ default:
+ status = DataBuffer::setProperty(type, value);
+ break;
+ }
+
+ return status;
+}
+
} /* namespace nntrainer */
return status;
}
-int DataBufferFromCallback::setFunc(
- BufferType type, std::function<bool(float *, float *, int *)> func) {
+int DataBufferFromCallback::setFunc(BufferType type, datagen_cb func) {
int status = ML_ERROR_NONE;
switch (type) {
bool *running = NULL;
std::vector<std::vector<float>> *data = NULL;
std::vector<std::vector<float>> *datalabel = NULL;
- std::function<bool(float *, float *, int *)> callback;
+ datagen_cb callback;
switch (type) {
case BUF_TRAIN: {
}
bool endflag = false;
+ float **vec_arr = (float **)malloc(sizeof(float *) * 1);
+ float **veclabel_arr = (float **)malloc(sizeof(float *) * 1);
+
float *vec =
(float *)malloc(sizeof(float) * input_dim.batch() * input_dim.channel() *
input_dim.height() * input_dim.width());
float *veclabel =
(float *)malloc(sizeof(float) * input_dim.batch() * class_num);
+ vec_arr[0] = vec;
+ veclabel_arr[0] = veclabel;
+
while ((*running)) {
trainReadyFlag = DATA_NOT_READY;
valReadyFlag = DATA_NOT_READY;
testReadyFlag = DATA_NOT_READY;
if (buf_size - (*cur_size) > 0) {
- endflag = callback(vec, veclabel, &status);
+ /** @todo Update to support multiple inputs later */
+ status = callback(vec_arr, veclabel_arr, &endflag);
- if (endflag) {
+ if (status == ML_ERROR_NONE && !endflag) {
for (unsigned int i = 0; i < input_dim.batch(); ++i) {
std::vector<float> v;
std::vector<float> vl;
}
}
- if (buf_size == (*cur_size) || !endflag) {
+ if (buf_size == (*cur_size) || endflag) {
switch (type) {
case BUF_TRAIN: {
std::lock_guard<std::mutex> lgtrain(readyTrainData);
- if (!endflag) {
+ if (status != ML_ERROR_NONE) {
+ trainReadyFlag = DATA_ERROR;
+ cv_train.notify_all();
+ free(vec);
+ free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
+ return;
+ } else if (endflag) {
trainReadyFlag = DATA_END;
cv_train.notify_all();
free(vec);
free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
return;
} else {
trainReadyFlag = DATA_READY;
} break;
case BUF_VAL: {
std::lock_guard<std::mutex> lgval(readyValData);
- if (!endflag) {
+ if (status != ML_ERROR_NONE) {
+ valReadyFlag = DATA_ERROR;
+ cv_val.notify_all();
+ free(vec);
+ free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
+ return;
+ } else if (endflag) {
valReadyFlag = DATA_END;
- cv_train.notify_all();
+ cv_val.notify_all();
free(vec);
free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
return;
} else {
valReadyFlag = DATA_READY;
} break;
case BUF_TEST: {
std::lock_guard<std::mutex> lgtest(readyTestData);
- if (!endflag) {
+ if (status != ML_ERROR_NONE) {
+ testReadyFlag = DATA_ERROR;
+ cv_test.notify_all();
+ free(vec);
+ free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
+ return;
+ } else if (endflag) {
testReadyFlag = DATA_END;
cv_test.notify_all();
free(vec);
free(veclabel);
+ free(vec_arr);
+ free(veclabel_arr);
return;
} else {
testReadyFlag = DATA_READY;
free(veclabel);
}
+int DataBufferFromCallback::setProperty(const PropertyType type,
+ std::string &value) {
+ return DataBuffer::setProperty(type, value);
+}
+
} /* namespace nntrainer */
unsigned int type = parseNetProperty(key);
+ /** TODO: disable this batch size */
switch (static_cast<PropertyType>(type)) {
case PropertyType::batch_size: {
status = setInt(batch_size, value);
NN_RETURN_STATUS();
epoch = e;
} break;
- case PropertyType::train_data: {
- status = std::static_pointer_cast<DataBufferFromDataFile>(data_buffer)
- ->setDataFile(value, DATA_TRAIN);
- NN_RETURN_STATUS();
- } break;
- case PropertyType::val_data: {
- status = std::static_pointer_cast<DataBufferFromDataFile>(data_buffer)
- ->setDataFile(value, DATA_VAL);
- NN_RETURN_STATUS();
- } break;
- case PropertyType::test_data: {
- status = std::static_pointer_cast<DataBufferFromDataFile>(data_buffer)
- ->setDataFile(value, DATA_TEST);
- NN_RETURN_STATUS();
- } break;
- case PropertyType::label_data: {
- status = std::static_pointer_cast<DataBufferFromDataFile>(data_buffer)
- ->setDataFile(value, DATA_LABEL);
- NN_RETURN_STATUS();
- } break;
-
- case PropertyType::buffer_size: {
- int size;
- status = setInt(size, value);
- NN_RETURN_STATUS();
- status = data_buffer->setBufSize(size);
- NN_RETURN_STATUS();
- } break;
case PropertyType::model_file: {
model = value;
} break;
int status = ML_ERROR_NONE;
if (data_buffer == nullptr) {
- data_buffer = std::make_shared<DataBufferFromDataFile>();
- }
-
- if (values.size() != 0) {
- status = data_buffer->setMiniBatch(layers[0]->getInputDimension().batch());
- NN_RETURN_STATUS();
-
- status = setTrainConfig(values);
- NN_RETURN_STATUS();
+ ml_loge("Cannot initialize the model without the data buffer.");
+ return ML_ERROR_INVALID_PARAMETER;
}
- status = data_buffer->setClassNum(
- layers[layers.size() - 1]->getOutputDimension().width());
- NN_RETURN_STATUS();
-
- status = data_buffer->setFeatureSize(layers[0]->getInputDimension());
- NN_RETURN_STATUS();
-
- status = data_buffer->init();
+ status = data_buffer->setMiniBatch(layers[0]->getInputDimension().batch());
NN_RETURN_STATUS();
- return train_run();
-}
-
-/**
- * @brief Run NeuralNetwork train
- */
-int NeuralNetwork::train(
- std::function<bool(float *, float *, int *)> train_func,
- std::function<bool(float *, float *, int *)> val_func,
- std::function<bool(float *, float *, int *)> test_func) {
-
- std::vector<std::string> values;
-
- return train(train_func, val_func, test_func, values);
-}
-
-/**
- * @brief Run NeuralNetwork train
- */
-int NeuralNetwork::train(
- std::function<bool(float *, float *, int *)> train_func,
- std::function<bool(float *, float *, int *)> val_func,
- std::function<bool(float *, float *, int *)> test_func,
- std::vector<std::string> values) {
-
- int status = ML_ERROR_NONE;
-
- if (data_buffer == nullptr) {
- data_buffer = std::make_shared<DataBufferFromCallback>();
-
- status = data_buffer->setMiniBatch(layers[0]->getInputDimension().batch());
- NN_RETURN_STATUS();
-
- status = setTrainConfig(values);
- NN_RETURN_STATUS();
- }
-
status = data_buffer->setClassNum(
layers[layers.size() - 1]->getOutputDimension().width());
NN_RETURN_STATUS();
status = data_buffer->init();
NN_RETURN_STATUS();
- std::shared_ptr<DataBufferFromCallback> callback_buffer =
- std::static_pointer_cast<DataBufferFromCallback>(data_buffer);
-
- status = callback_buffer->setFunc(nntrainer::BUF_TRAIN, (train_func));
- if (status != ML_ERROR_NONE)
- return status;
-
- status = callback_buffer->setFunc(nntrainer::BUF_VAL, (val_func));
- if (status != ML_ERROR_NONE)
- return status;
-
- status = callback_buffer->setFunc(nntrainer::BUF_TEST, (test_func));
- if (status != ML_ERROR_NONE)
- return status;
+ status = setTrainConfig(values);
+ NN_RETURN_STATUS();
return train_run();
-};
+}
/**
* @brief Run NeuralNetwork train with callback function by user
return ML_ERROR_NONE;
}
+int NeuralNetwork::setDataBuffer(std::shared_ptr<DataBuffer> data_buffer) {
+ this->data_buffer = data_buffer;
+
+ return ML_ERROR_NONE;
+}
+
void NeuralNetwork::ensureName(std::shared_ptr<Layer> layer,
std::string prefix) {
if (layer->getName().empty()) {
* @brief Network Properties
* loss = 0,
* cost = 1,
- * train_data = 2,
- * val_data = 3,
- * test_data = 4,
- * label_data = 5,
- * buffer_size = 6,
- * batch_size = 7,
- * epochs = 8,
- * model_file = 9
+ * batch_size = 2,
+ * epochs = 3,
+ * model_file = 4
*/
- std::array<std::string, 10> property_string = {
- "loss", "cost", "train_data", "val_data", "test_data",
- "label_data", "buffer_size", "batch_size", "epochs", "model_file"};
+ std::array<std::string, 5> property_string = {"loss", "cost", "batch_size",
+ "epochs", "model_file"};
for (i = 0; i < property_string.size(); i++) {
unsigned int size = (property_string[i].size() > property.size())
return (unsigned int)NeuralNetwork::PropertyType::unknown;
}
+unsigned int parseDataProperty(std::string property) {
+ unsigned int i;
+
+ /**
+ * @brief Network Properties
+ * train_data = 0,
+ * val_data = 1,
+ * test_data = 2,
+ * label_data = 3,
+ * buffer_size = 4
+ */
+ std::array<std::string, 5> property_string = {
+ "train_data", "val_data", "test_data", "label_data", "buffer_size"};
+
+ for (i = 0; i < property_string.size(); i++) {
+ unsigned int size = (property_string[i].size() > property.size())
+ ? property_string[i].size()
+ : property.size();
+
+ if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) {
+ return (i);
+ }
+ }
+
+ return (unsigned int)DataBuffer::PropertyType::unknown;
+}
+
int setInt(int &val, std::string str) {
int status = ML_ERROR_NONE;
try {
* @brief get data which size is mini batch for train
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_train(float *outVec, float *outLabel, int *status);
+int getMiniBatch_train(float **outVec, float **outLabel, bool *last);
/**
* @brief get data which size is mini batch for val
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_val(float *outVec, float *outLabel, int *status);
+int getMiniBatch_val(float **outVec, float **outLabel, bool *last);
#endif /* __cplusplus */
#endif /* __NNTRAINER_TEST_UTIL_H__ */
#include "nntrainer_test_util.h"
#include <climits>
#include <iostream>
+#include <nntrainer_error.h>
#include <random>
#include <tensor.h>
* @brief get data which size is mini batch for train
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_train(float *outVec, float *outLabel, int *status) {
+int getMiniBatch_train(float **outVec, float **outLabel, bool *last) {
std::vector<int> memI;
std::vector<int> memJ;
unsigned int count = 0;
unsigned int data_size = 0;
+ *last = true;
std::string filename = "trainingSet.dat";
std::ifstream F(filename, std::ios::in | std::ios::binary);
if (count < mini_batch) {
free(duplicate);
alloc_train = false;
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
getData(F, o, l, memI[i]);
for (unsigned int j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (unsigned int j = 0; j < num_class; ++j)
- outLabel[i * num_class + j] = l[j];
+ outLabel[0][i * num_class + j] = l[j];
}
F.close();
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
* @brief get data which size is mini batch for validation
* @param[out] outVec
* @param[out] outLabel
- * @param[out] status for error handling
- * @retval true/false false : end of data
+ * @param[out] last if the data is finished
+ * @retval status for handling error
*/
-bool getMiniBatch_val(float *outVec, float *outLabel, int *status) {
+int getMiniBatch_val(float **outVec, float **outLabel, bool *last) {
std::vector<int> memI;
std::vector<int> memJ;
unsigned int count = 0;
unsigned int data_size = 0;
+ *last = true;
std::string filename = "trainingSet.dat";
std::ifstream F(filename, std::ios::in | std::ios::binary);
if (count < mini_batch) {
free(valduplicate);
alloc_val = false;
- return false;
+ *last = true;
+ return ML_ERROR_NONE;
}
count = 0;
getData(F, o, l, memI[i]);
for (unsigned int j = 0; j < feature_size; ++j)
- outVec[i * feature_size + j] = o[j];
+ outVec[0][i * feature_size + j] = o[j];
for (unsigned int j = 0; j < num_class; ++j)
- outLabel[i * num_class + j] = l[j];
+ outLabel[0][i * num_class + j] = l[j];
}
F.close();
- return true;
+ *last = false;
+ return ML_ERROR_NONE;
}
/**
int status = ML_ERROR_NONE;
std::string config_file = "./test_train_01_p.ini";
RESET_CONFIG(config_file.c_str());
- replaceString("Input_Shape = 32:1:1:62720", "Input_Shape=32:1:1:62720",
+ replaceString("Input_Shape = 32:1:1:62720", "Input_Shape=16:1:1:62720",
config_file, config_str);
replaceString("minibatch = 32", "minibatch = 16", config_file, config_str);
replaceString("BufferSize=100", "", config_file, config_str);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_compile(handle, NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = ml_nnmodel_train_with_file(handle, NULL);
+ status = ml_train_model_run(handle, NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_destroy(handle);
EXPECT_EQ(status, ML_ERROR_NONE);
*/
TEST(nntrainer_capi_nnmodel, train_02_n) {
int status = ML_ERROR_NONE;
- status = ml_nnmodel_train_with_file(NULL, NULL);
+ status = ml_train_model_run(NULL, NULL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
}
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_compile(handle, NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = ml_nnmodel_train_with_file(handle, "loss=cross", NULL);
+ status = ml_train_model_run(handle, "loss=cross", NULL);
EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER);
status = ml_train_model_destroy(handle);
EXPECT_EQ(status, ML_ERROR_NONE);
ml_train_model_h model;
ml_train_layer_h layers[2];
ml_train_optimizer_h optimizer;
+ ml_train_dataset_h dataset;
status = ml_train_model_construct(&model);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_set_optimizer(model, optimizer);
EXPECT_EQ(status, ML_ERROR_NONE);
+ status = ml_train_dataset_create_with_file(&dataset, "trainingSet.dat",
+ "valSet.dat", NULL);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+
+ status = ml_train_dataset_set_property(dataset, "label_data=label.dat",
+ "buffer_size=100", NULL);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+
+ status = ml_train_model_set_dataset(model, dataset);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+
status = ml_train_model_compile(model, "loss=cross", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = ml_nnmodel_train_with_file(
- model, "epochs=2", "batch_size=16", "train_data=trainingSet.dat",
- "val_data=valSet.dat", "label_data=label.dat", "buffer_size=100",
- "model_file=model.bin", NULL);
+ status = ml_train_model_run(model, "epochs=2", "batch_size=16",
+ "model_file=model.bin", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_destroy(model);
ml_train_model_h model;
ml_train_layer_h layers[2];
ml_train_optimizer_h optimizer;
+ ml_train_dataset_h dataset;
status = ml_train_model_construct(&model);
EXPECT_EQ(status, ML_ERROR_NONE);
status = ml_train_model_set_optimizer(model, optimizer);
EXPECT_EQ(status, ML_ERROR_NONE);
+ status = ml_train_dataset_create_with_generator(&dataset, getMiniBatch_train,
+ getMiniBatch_val, NULL);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+
+ status = ml_train_dataset_set_property(dataset, "buffer_size=100", NULL);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+
+ status = ml_train_model_set_dataset(model, dataset);
+ EXPECT_EQ(status, ML_ERROR_NONE);
+
status = ml_train_model_compile(model, "loss=cross", NULL);
EXPECT_EQ(status, ML_ERROR_NONE);
- status = ml_nnmodel_train_with_generator(
- model, getMiniBatch_train, getMiniBatch_val, NULL, "epochs=2",
- "batch_size=16", "buffer_size=100", "model_file=model.bin", NULL);
+ status = ml_train_model_run(model, "epochs=2", "batch_size=16",
+ "model_file=model.bin", NULL);
status = ml_train_model_destroy(model);
EXPECT_EQ(status, ML_ERROR_NONE);
projects / platform / core / ml / nntrainer.git / commitdiff