From 51015b922f7aff76a4d6fe78c8170a1ce718cd9f Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Tue, 14 Jul 2020 13:30:09 +0900 Subject: [PATCH] [API] Update C-API dataset handling Update C-API to add dataset handling Changes added to data buffer: - Bug fix for data buffer using already freed memory - Data buffer set properties moved out from neural network - Data buffer generator now takes list of array for multiple inputs Signed-off-by: Parichay Kapoor --- Applications/Classification/jni/Android.mk | 6 +- Applications/Classification/jni/main_func.cpp | 46 +++-- Applications/Tizen_CAPI/capi_file.c | 23 ++- Applications/Tizen_CAPI/capi_func.c | 55 ++++-- Applications/Tizen_CAPI/main.c | 2 +- Applications/Training/jni/Android.mk | 2 +- Applications/mnist/jni/Android.mk | 4 +- Applications/mnist/jni/main.cpp | 49 ++++-- api/capi/include/nntrainer.h | 131 +++++++++++--- api/capi/include/nntrainer_internal.h | 93 +++++----- api/capi/src/nntrainer.cpp | 238 ++++++++++++++++++++------ jni/Android.mk | 3 +- meson.build | 2 +- nntrainer/include/databuffer.h | 51 +++++- nntrainer/include/databuffer_file.h | 11 +- nntrainer/include/databuffer_func.h | 28 ++- nntrainer/include/neuralnet.h | 35 +--- nntrainer/include/parse_util.h | 7 + nntrainer/meson.build | 3 +- nntrainer/src/bn_layer.cpp | 1 - nntrainer/src/databuffer.cpp | 41 +++++ nntrainer/src/databuffer_file.cpp | 28 +++ nntrainer/src/databuffer_func.cpp | 61 +++++-- nntrainer/src/neuralnet.cpp | 110 ++---------- nntrainer/src/parse_util.cpp | 43 +++-- test/include/nntrainer_test_util.h | 12 +- test/nntrainer_test_util.cpp | 35 ++-- test/tizen_capi/unittest_tizen_capi.cpp | 42 +++-- 28 files changed, 794 insertions(+), 368 deletions(-) diff --git a/Applications/Classification/jni/Android.mk b/Applications/Classification/jni/Android.mk index f77288b..e7b9398 100644 --- a/Applications/Classification/jni/Android.mk +++ b/Applications/Classification/jni/Android.mk @@ -9,7 +9,9 @@ endif ifndef NNTRAINER_ROOT NNTRAINER_ROOT := $(LOCAL_PATH)/../../../libs/arm64-v8a -NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include +NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include \ + $(LOCAL_PATH)/../../../api/capi/include \ + $(LOCAL_PATH)/../../../api/capi/include/platform endif NNTRAINER_APPLICATION := $(LOCAL_PATH)/../.. @@ -45,7 +47,7 @@ include $(PREBUILT_SHARED_LIBRARY) include $(CLEAR_VARS) LOCAL_ARM_NEON := true -LOCAL_CFLAGS += -std=c++14 -Ofast -mcpu=cortex-a53 -Ilz4-nougat/lib +LOCAL_CFLAGS += -std=c++14 -Ofast -mcpu=cortex-a53 -Ilz4-nougat/lib LOCAL_LDFLAGS += -Llz4-nougat/lib/obj/local/arm64-v8a/ LOCAL_CXXFLAGS += -std=c++14 LOCAL_CFLAGS += -pthread -fopenmp -fexceptions diff --git a/Applications/Classification/jni/main_func.cpp b/Applications/Classification/jni/main_func.cpp index 9045dc7..79ce496 100644 --- a/Applications/Classification/jni/main_func.cpp +++ b/Applications/Classification/jni/main_func.cpp @@ -33,7 +33,10 @@ #include #include +#include "databuffer.h" +#include "databuffer_func.h" #include "neuralnet.h" +#include "nntrainer_error.h" #include "tensor.h" #define TRAINING true @@ -137,10 +140,10 @@ bool getData(std::ifstream &F, std::vector &outVec, * @brief get data which size is mini batch for train * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { +int getMiniBatch_train(float **outVec, float **outLabel, bool *last) { std::vector memI; std::vector memJ; unsigned int count = 0; @@ -158,7 +161,8 @@ bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { for (unsigned int i = 0; i < total_label_size * data_size; ++i) { duplicate[i] = false; } - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -181,23 +185,24 @@ bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { getData(F, o, l, memI[i]); for (unsigned int j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (unsigned int j = 0; j < total_label_size; ++j) - outLabel[i * total_label_size + j] = l[j]; + outLabel[0][i * total_label_size + j] = l[j]; } F.close(); - return true; + *last = false; + return ML_ERROR_NONE; } /** * @brief get data which size is mini batch for validation * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false false : end of data + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { +int getMiniBatch_val(float **outVec, float **outLabel, bool *last) { std::vector memI; std::vector memJ; @@ -216,7 +221,8 @@ bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { for (unsigned int i = 0; i < total_label_size * data_size; ++i) { valduplicate[i] = false; } - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -239,13 +245,14 @@ bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { getData(F, o, l, memI[i]); for (unsigned int j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (unsigned int j = 0; j < total_label_size; ++j) - outLabel[i * total_label_size + j] = l[j]; + outLabel[0][i * total_label_size + j] = l[j]; } F.close(); - return true; + *last = false; + return ML_ERROR_NONE; } /** @@ -277,6 +284,14 @@ int main(int argc, char *argv[]) { } /** + * @brief Data buffer Create & Initialization + */ + std::shared_ptr DB = + std::make_shared(); + DB->setFunc(nntrainer::BUF_TRAIN, getMiniBatch_train); + DB->setFunc(nntrainer::BUF_VAL, getMiniBatch_val); + + /** * @brief Neural Network Create & Initialization */ nntrainer::NeuralNetwork NN; @@ -284,11 +299,12 @@ int main(int argc, char *argv[]) { NN.loadFromConfig(); NN.init(); NN.readModel(); + NN.setDataBuffer((DB)); /** * @brief Neural Network Train & validation */ - NN.train(getMiniBatch_train, getMiniBatch_val, nullptr); + NN.train(); /** * @brief Finalize NN diff --git a/Applications/Tizen_CAPI/capi_file.c b/Applications/Tizen_CAPI/capi_file.c index f45c77f..9f6c8a8 100644 --- a/Applications/Tizen_CAPI/capi_file.c +++ b/Applications/Tizen_CAPI/capi_file.c @@ -38,10 +38,11 @@ int main(int argc, char *argv[]) { int status = ML_ERROR_NONE; - /* handlers for model, layers & optimizer */ + /* handlers for model, layers, optimizer and dataset */ ml_train_model_h model; ml_train_layer_h layers[2]; ml_train_optimizer_h optimizer; + ml_train_dataset_h dataset; /* model create */ status = ml_train_model_construct(&model); @@ -93,12 +94,24 @@ int main(int argc, char *argv[]) { status = ml_train_model_compile(model, "loss=cross", NULL); NN_RETURN_STATUS(); + /* create dataset */ + status = ml_train_dataset_create_with_file(&dataset, "trainingSet.dat", + "valSet.dat", NULL); + NN_RETURN_STATUS(); + + /* set property for dataset */ + status = ml_train_dataset_set_property(dataset, "label_data=label.dat", + "buffer_size=100", NULL); + NN_RETURN_STATUS(); + + /* set dataset */ + status = ml_train_model_set_dataset(model, dataset); + NN_RETURN_STATUS(); + /* train model with data files : epochs = 10 and store model file named * "model.bin" */ - status = ml_nnmodel_train_with_file( - model, "epochs=10", "batch_size=32", "train_data=trainingSet.dat", - "val_data=trainingSet.dat", "label_data=label.dat", "buffer_size=100", - "model_file=model.bin", NULL); + status = ml_train_model_run(model, "epochs=10", "batch_size=32", + "model_file=model.bin", NULL); NN_RETURN_STATUS(); /* delete model */ diff --git a/Applications/Tizen_CAPI/capi_func.c b/Applications/Tizen_CAPI/capi_func.c index 1e82cf0..d5091b4 100644 --- a/Applications/Tizen_CAPI/capi_func.c +++ b/Applications/Tizen_CAPI/capi_func.c @@ -31,8 +31,8 @@ static bool *valduplicate; static bool alloc_train = false; static bool alloc_val = false; -bool gen_data_train(float *outVec, float *outLabel, int *status); -bool gen_data_val(float *outVec, float *outLabel, int *status); +int gen_data_train(float **outVec, float **outLabel, bool *last); +int gen_data_val(float **outVec, float **outLabel, bool *last); bool file_exists(const char *filename); bool file_exists(const char *filename) { @@ -120,7 +120,7 @@ static bool get_data(const char *file_name, float *outVec, float *outLabel, * @param[out] status for error handling * @retval true/false */ -bool gen_data_train(float *outVec, float *outLabel, int *status) { +int gen_data_train(float **outVec, float **outLabel, bool *last) { int memI[mini_batch]; long file_size; unsigned int count = 0; @@ -132,7 +132,7 @@ bool gen_data_train(float *outVec, float *outLabel, int *status) { if (!file_exists(file_name)) { printf("%s does not exists\n", file_name); - return false; + return ML_ERROR_INVALID_PARAMETER; } file = fopen(file_name, "r"); @@ -162,7 +162,8 @@ bool gen_data_train(float *outVec, float *outLabel, int *status) { } free(duplicate); alloc_train = false; - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -182,22 +183,23 @@ bool gen_data_train(float *outVec, float *outLabel, int *status) { get_data(file_name, o, l, memI[i], file_size); for (j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (j = 0; j < num_class; ++j) - outLabel[i * num_class + j] = l[j]; + outLabel[0][i * num_class + j] = l[j]; } - return true; + *last = false; + return ML_ERROR_NONE; } /** * @brief get data which size is mini batch for validation * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false false : end of data + * @param[out] last if the data is finished + * @retval status for handling error */ -bool gen_data_val(float *outVec, float *outLabel, int *status) { +int gen_data_val(float **outVec, float **outLabel, bool *last) { int memI[mini_batch]; unsigned int i, j; @@ -230,7 +232,8 @@ bool gen_data_val(float *outVec, float *outLabel, int *status) { if (count < mini_batch) { free(valduplicate); alloc_val = false; - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -250,22 +253,24 @@ bool gen_data_val(float *outVec, float *outLabel, int *status) { get_data(file_name, o, l, memI[i], file_size); for (j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (j = 0; j < num_class; ++j) - outLabel[i * num_class + j] = l[j]; + outLabel[0][i * num_class + j] = l[j]; } - return true; + *last = false; + return ML_ERROR_NONE; } int main(int argc, char *argv[]) { int status = ML_ERROR_NONE; - /* handlers for model, layers & optimizer */ + /* handlers for model, layers, optimizer and dataset */ ml_train_model_h model; ml_train_layer_h layers[2]; ml_train_optimizer_h optimizer; + ml_train_dataset_h dataset; /* model create */ status = ml_train_model_construct(&model); @@ -317,11 +322,23 @@ int main(int argc, char *argv[]) { status = ml_train_model_compile(model, "loss=cross", NULL); NN_RETURN_STATUS(); + /* create dataset */ + status = ml_train_dataset_create_with_generator(&dataset, gen_data_train, + gen_data_val, NULL); + NN_RETURN_STATUS(); + + /* set property for dataset */ + status = ml_train_dataset_set_property(dataset, "buffer_size=32", NULL); + NN_RETURN_STATUS(); + + /* set dataset */ + status = ml_train_model_set_dataset(model, dataset); + NN_RETURN_STATUS(); + /* train model with data files : epochs = 10 and store model file named * "model.bin" */ - status = ml_nnmodel_train_with_generator( - model, gen_data_train, gen_data_val, NULL, "epochs=10", "batch_size=32", - "model_file=model.bin", "buffer_size = 32", NULL); + status = ml_train_model_run(model, "epochs=10", "batch_size=32", + "model_file=model.bin", NULL); NN_RETURN_STATUS(); /* delete model */ diff --git a/Applications/Tizen_CAPI/main.c b/Applications/Tizen_CAPI/main.c index bb5c625..462ef17 100644 --- a/Applications/Tizen_CAPI/main.c +++ b/Applications/Tizen_CAPI/main.c @@ -35,7 +35,7 @@ int main(int argc, char *argv[]) { status = ml_train_model_compile(handle, NULL); if (status != ML_ERROR_NONE) return status; - status = ml_nnmodel_train_with_file(handle, NULL); + status = ml_train_model_run(handle, NULL); if (status != ML_ERROR_NONE) return status; status = ml_train_model_destroy(handle); diff --git a/Applications/Training/jni/Android.mk b/Applications/Training/jni/Android.mk index 1d40a44..1abc404 100644 --- a/Applications/Training/jni/Android.mk +++ b/Applications/Training/jni/Android.mk @@ -60,7 +60,7 @@ LOCAL_SRC_FILES := main.cpp bitmap_helpers.cpp LOCAL_SHARED_LIBRARIES := nntrainer -LOCAL_STATIC_LIBRARIES := tensorflow-lite +LOCAL_STATIC_LIBRARIES := tensorflow-lite LOCAL_C_INCLUDES += $(TFLITE_INCLUDES) $(NNTRAINER_INCLUDES) diff --git a/Applications/mnist/jni/Android.mk b/Applications/mnist/jni/Android.mk index 9b53d3a..403f4d6 100644 --- a/Applications/mnist/jni/Android.mk +++ b/Applications/mnist/jni/Android.mk @@ -9,7 +9,9 @@ endif ifndef NNTRAINER_ROOT NNTRAINER_ROOT := $(LOCAL_PATH)/../../../libs/arm64-v8a -NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include +NNTRAINER_INCLUDES := $(LOCAL_PATH)/../../../nntrainer/include \ + $(LOCAL_PATH)/../../../api/capi/include \ + $(LOCAL_PATH)/../../../api/capi/include/platform endif include $(CLEAR_VARS) diff --git a/Applications/mnist/jni/main.cpp b/Applications/mnist/jni/main.cpp index 2cf9e1f..9daac19 100644 --- a/Applications/mnist/jni/main.cpp +++ b/Applications/mnist/jni/main.cpp @@ -30,7 +30,10 @@ #include #include +#include "databuffer.h" +#include "databuffer_func.h" #include "neuralnet.h" +#include "nntrainer_error.h" #include "tensor.h" #define TRAINING true @@ -131,10 +134,10 @@ bool getData(std::ifstream &F, std::vector &outVec, * @brief get data which size is mini batch for train * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { +int getMiniBatch_train(float **outVec, float **outLabel, bool *last) { std::vector memI; std::vector memJ; unsigned int count = 0; @@ -152,7 +155,8 @@ bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { for (unsigned int i = 0; i < total_label_size * data_size; ++i) { duplicate[i] = false; } - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -175,23 +179,24 @@ bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { getData(F, o, l, memI[i]); for (unsigned int j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (unsigned int j = 0; j < total_label_size; ++j) - outLabel[i * total_label_size + j] = l[j]; + outLabel[0][i * total_label_size + j] = l[j]; } F.close(); - return true; + *last = false; + return ML_ERROR_NONE; } /** * @brief get data which size is mini batch for validation * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false false : end of data + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { +int getMiniBatch_val(float **outVec, float **outLabel, bool *last) { std::vector memI; std::vector memJ; @@ -210,7 +215,8 @@ bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { for (unsigned int i = 0; i < total_label_size * data_size; ++i) { valduplicate[i] = false; } - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -233,13 +239,14 @@ bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { getData(F, o, l, memI[i]); for (unsigned int j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (unsigned int j = 0; j < total_label_size; ++j) - outLabel[i * total_label_size + j] = l[j]; + outLabel[0][i * total_label_size + j] = l[j]; } F.close(); - return true; + *last = false; + return ML_ERROR_NONE; } /** @@ -271,17 +278,27 @@ int main(int argc, char *argv[]) { } /** + * @brief Data buffer Create & Initialization + */ + std::shared_ptr DB = + std::make_shared(); + DB->setFunc(nntrainer::BUF_TRAIN, getMiniBatch_train); + DB->setFunc(nntrainer::BUF_VAL, getMiniBatch_val); + + /** * @brief Neural Network Create & Initialization */ - nntrainer::NeuralNetwork NN(config); + nntrainer::NeuralNetwork NN; + NN.setConfig(config); NN.loadFromConfig(); NN.init(); NN.readModel(); + NN.setDataBuffer((DB)); /** * @brief Neural Network Train & validation */ - NN.train(getMiniBatch_train, getMiniBatch_val, nullptr); + NN.train(); /** * @brief Finalize NN diff --git a/api/capi/include/nntrainer.h b/api/capi/include/nntrainer.h index d0b0fd6..fefaaa3 100644 --- a/api/capi/include/nntrainer.h +++ b/api/capi/include/nntrainer.h @@ -56,6 +56,12 @@ typedef void *ml_train_layer_h; typedef void *ml_train_optimizer_h; /** + * @brief A handle of an NNTrainer dataset. + * @since_tizen 6.x + */ +typedef void *ml_train_dataset_h; + +/** * @brief Enumeration for the neural network layer type of NNTrainer. * @since_tizen 6.x */ @@ -88,6 +94,26 @@ typedef enum { } ml_train_summary_type_e; /** + * @brief Dataset generator callback function for train/valid/test data. + * @details The Containers passed will already be allocated with sufficient + * space to contain the data by the caller. This function should return a batch + * of input and label of data in the passed containers. The callback should fill + * the data row-wise in the containers obliging to the input shape set for the + * model. The order of the inputs in case of multiple input layers will be + * determined based on the sequence of addition of the input layers to the + * model. + * @note This function can be called multiple times in parallel. + * @param[out] input Container to hold all the input data. + * @param[out] label Container to hold corresponding label data. + * @param[out] last Container to notify if data is finished. Set true if no more + * data to provide, else set false. + * @return @c 0 on success. Otherwise a negative error value. + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. + */ +typedef int (*ml_train_datagen_cb)(float **input, float **label, bool *last); + +/** * @brief Constructs the neural network model. * @details Use this function to create Neural Netowrk Model. * @since_tizen 6.x @@ -128,35 +154,18 @@ int ml_train_model_construct_with_conf(const char *model_conf, int ml_train_model_compile(ml_train_model_h model, ...); /** - * @brief train the neural network model. - * @details Use this function to train neural network model - * @since_tizen 6.x - * @param[in] model The NNTrainer model handler from the given description. - * @param[in] ... hyper parmeter for train model - * @return @c 0 on success. Otherwise a negative error value. - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. - */ -int ml_nnmodel_train_with_file(ml_train_model_h model, ...); - -/** - * @brief train the neural network model. - * @details Use this function to train neural network model + * @brief Train the neural network model. + * @details Use this function to train the compiled neural network model with + * the passed training hyperparameters. This function will return once the + * training along with requested validation and testing is completed. * @since_tizen 6.x * @param[in] model The NNTrainer model handler from the given description. - * @param[in] train_func function pointer for train - * @param[in] val_func function pointer for val - * @param[in] test_func function pointer for test * @param[in] ... hyper parmeter for train model * @return @c 0 on success. Otherwise a negative error value. * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. */ -int ml_nnmodel_train_with_generator(ml_train_model_h model, - bool (*train_func)(float *, float *, int *), - bool (*val_func)(float *, float *, int *), - bool (*test_func)(float *, float *, int *), - ...); +int ml_train_model_run(ml_train_model_h model, ...); /** * @brief Destructs the neural network model. @@ -217,6 +226,22 @@ int ml_train_model_set_optimizer(ml_train_model_h model, ml_train_optimizer_h optimizer); /** + * @brief Set the dataset (data provider) for the neural network model. + * @details Use this function to set dataset for running the model. The dataset + * will provide training, validation and test data for the model. Unsets the + * previous dataset if any. This transfers the ownership of the dataset to + * the network. No need to delete the dataset once it is set to a model. + * @since_tizen 6.x + * @param[in] model The NNTrainer model handler. + * @param[in] dataset The NNTrainer dataset handler. + * @return @c 0 on success. Otherwise a negative error value. + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. + */ +int ml_train_model_set_dataset(ml_train_model_h model, + ml_train_dataset_h dataset); + +/** * @brief Create the neural network layer. * @details Use this function to create Neural Netowrk Layer. * @since_tizen 6.x @@ -293,6 +318,68 @@ int ml_train_optimizer_destroy(ml_train_optimizer_h optimizer); int ml_train_optimizer_set_property(ml_train_optimizer_h optimizer, ...); /** + * @brief Create a dataset with generators to feed to a neural network. + * @details Use this function to create a Neural Network Dataset using + * generators. The generators will provide data representing a single input + * batch. When setting this dataset to a model, the data generated by the + * generators should match the input and the label shape for the model. + * @since_tizen 6.x + * @param[out] dataset The NNTrainer Dataset handler from the given description. + * @param[in] train_cb The dataset generator for training. + * @param[in] valid_cb The dataset generator for validating. Can be null. + * @param[in] test_cb The dataset generator for testing. Can be null. + * @return @c 0 on success. Otherwise a negative error value. + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. + */ +int ml_train_dataset_create_with_generator(ml_train_dataset_h *dataset, + ml_train_datagen_cb train_cb, + ml_train_datagen_cb valid_cb, + ml_train_datagen_cb test_cb); + +/** + * @brief Create a dataset with files to feed to a neural network. + * @details Use this function to create a Neural Network Dataset using + * files. + * @since_tizen 6.x + * @param[out] dataset The NNTrainer Dataset handler from the given description. + * @param[in] train_fle The dataset file for training. + * @param[in] valid_file The dataset file for validating. Can be null. + * @param[in] test_file The dataset file for testing. Can be null. + * @return @c 0 on success. Otherwise a negative error value. + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. + */ +int ml_train_dataset_create_with_file(ml_train_dataset_h *dataset, + const char *train_file, + const char *valid_file, + const char *test_file); + +/** + * @brief Destroy the neural network dataset. + * @details Use this function to destroy dataset. Fails if dataset is owned by a + * model. + * @since_tizen 6.x + * @param[in] dataset The NNTrainer dataset handler. + * @return @c 0 on success. Otherwise a negative error value. + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER Invalid Parameter. + */ +int ml_train_dataset_destroy(ml_train_dataset_h dataset); + +/** + * @brief Set the neural network dataset Property. + * @details Use this function to set dataset Property. + * @since_tizen 6.x + * @param[in] dataset The NNTrainer dataset handler. + * @param[in] ... Property values with NULL for termination. + * @return @c 0 on success. Otherwise a negative error value. + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER Invalid parameter. + */ +int ml_train_dataset_set_property(ml_train_dataset_h dataset, ...); + +/** * @} */ #ifdef __cplusplus diff --git a/api/capi/include/nntrainer_internal.h b/api/capi/include/nntrainer_internal.h index bdac119..fbee2b7 100644 --- a/api/capi/include/nntrainer_internal.h +++ b/api/capi/include/nntrainer_internal.h @@ -13,11 +13,12 @@ */ /** * @file nntrainer_internal.h - * @date 02 April 2020 + * @date 13 April 2020 * @brief NNTrainer C-API Internal Header. * This allows to construct and control NNTrainer Model. * @see https://github.com/nnstreamer/nntrainer * @author Jijoong Moon + * @author Parichay Kapoor * @bug No known bugs except for NYI items */ @@ -38,63 +39,71 @@ extern "C" { #endif /* __cplusplus */ +/** + * @brief Struct to wrap neural network layer for the API + */ typedef struct { uint magic; std::shared_ptr layer; bool in_use; -} ml_nnlayer; +} ml_train_layer; +/** + * @brief Struct to wrap neural network optimizer for the API + */ typedef struct { uint magic; std::shared_ptr optimizer; bool in_use; -} ml_nnopt; +} ml_train_optimizer; +/** + * @brief Struct to wrap data buffer for the API + */ +typedef struct { + uint magic; + std::shared_ptr data_buffer; + bool in_use; +} ml_train_dataset; + +/** + * @brief Struct to wrap neural network model for the API + */ typedef struct { uint magic; std::shared_ptr network; - std::unordered_map layers_map; - ml_nnopt *optimizer; -} ml_nnmodel; - -#define ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model) \ - do { \ - if (!model) { \ - ml_loge("Error: Invalid Parameter : model is empty."); \ - return ML_ERROR_INVALID_PARAMETER; \ - } \ - nnmodel = (ml_nnmodel *)model; \ - if (nnmodel->magic != ML_NNTRAINER_MAGIC) { \ - ml_loge("Error: Invalid Parameter : nnmodel is invalid."); \ - return ML_ERROR_INVALID_PARAMETER; \ - } \ - } while (0) + std::unordered_map layers_map; + ml_train_optimizer *optimizer; + ml_train_dataset *dataset; +} ml_train_model; -#define ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer) \ - do { \ - if (!layer) { \ - ml_loge("Error: Invalid Parameter : layer is empty."); \ - return ML_ERROR_INVALID_PARAMETER; \ - } \ - nnlayer = (ml_nnlayer *)layer; \ - if (nnlayer->magic != ML_NNTRAINER_MAGIC) { \ - ml_loge("Error: Invalid Parameter : nnlayer is invalid."); \ - return ML_ERROR_INVALID_PARAMETER; \ - } \ +/** + * @brief Check validity of the user passed arguments + */ +#define ML_NNTRAINER_CHECK_VALIDATION(obj, obj_h, obj_type, obj_name) \ + do { \ + if (!obj_h) { \ + ml_loge("Error: Invalid Parameter : %s is empty.", obj_name); \ + return ML_ERROR_INVALID_PARAMETER; \ + } \ + obj = (obj_type *)obj_h; \ + if (obj->magic != ML_NNTRAINER_MAGIC) { \ + ml_loge("Error: Invalid Parameter : %s is invalid.", obj_name); \ + return ML_ERROR_INVALID_PARAMETER; \ + } \ } while (0) -#define ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, opt) \ - do { \ - if (!opt) { \ - ml_loge("Error: Invalid Parameter : optimizer is empty."); \ - return ML_ERROR_INVALID_PARAMETER; \ - } \ - nnopt = (ml_nnopt *)opt; \ - if (nnopt->magic != ML_NNTRAINER_MAGIC) { \ - ml_loge("Error: Invalid Parameter : nnoptimizer is invalid."); \ - return ML_ERROR_INVALID_PARAMETER; \ - } \ - } while (0) +#define ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model) \ + ML_NNTRAINER_CHECK_VALIDATION(nnmodel, model, ml_train_model, "model") + +#define ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer) \ + ML_NNTRAINER_CHECK_VALIDATION(nnlayer, layer, ml_train_layer, "layer") + +#define ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, opt) \ + ML_NNTRAINER_CHECK_VALIDATION(nnopt, opt, ml_train_optimizer, "optimizer") + +#define ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset) \ + ML_NNTRAINER_CHECK_VALIDATION(nndataset, dataset, ml_train_dataset, "dataset") /** * @brief Get neural network layer from the model with the given name. diff --git a/api/capi/src/nntrainer.cpp b/api/capi/src/nntrainer.cpp index 5d1a67a..b159880 100644 --- a/api/capi/src/nntrainer.cpp +++ b/api/capi/src/nntrainer.cpp @@ -20,6 +20,9 @@ * @author Jijoong Moon * @bug No known bugs except for NYI items */ +#include +#include +#include #include #include #include @@ -73,9 +76,10 @@ static int nn_object(ml_train_model_h *model) { if (model == NULL) return ML_ERROR_INVALID_PARAMETER; - ml_nnmodel *nnmodel = new ml_nnmodel; + ml_train_model *nnmodel = new ml_train_model; nnmodel->magic = ML_NNTRAINER_MAGIC; nnmodel->optimizer = NULL; + nnmodel->dataset = NULL; *model = nnmodel; @@ -101,7 +105,7 @@ int ml_train_model_construct(ml_train_model_h *model) { int ml_train_model_construct_with_conf(const char *model_conf, ml_train_model_h *model) { int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; + ml_train_model *nnmodel; std::shared_ptr NN; returnable f; @@ -115,7 +119,7 @@ int ml_train_model_construct_with_conf(const char *model_conf, if (status != ML_ERROR_NONE) return status; - nnmodel = (ml_nnmodel *)(*model); + nnmodel = (ml_train_model *)(*model); NN = nnmodel->network; f = [&]() { return NN->setConfig(model_conf); }; @@ -137,7 +141,7 @@ int ml_train_model_construct_with_conf(const char *model_conf, int ml_train_model_compile(ml_train_model_h model, ...) { int status = ML_ERROR_NONE; const char *data; - ml_nnmodel *nnmodel; + ml_train_model *nnmodel; returnable f; ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); @@ -170,9 +174,9 @@ int ml_train_model_compile(ml_train_model_h model, ...) { return status; } -int ml_nnmodel_train_with_file(ml_train_model_h model, ...) { +int ml_train_model_run(ml_train_model_h model, ...) { int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; + ml_train_model *nnmodel; const char *data; std::vector arg_list; @@ -194,40 +198,9 @@ int ml_nnmodel_train_with_file(ml_train_model_h model, ...) { return status; } -int ml_nnmodel_train_with_generator(ml_train_model_h model, - bool (*train_func)(float *, float *, int *), - bool (*val_func)(float *, float *, int *), - bool (*test_func)(float *, float *, int *), - ...) { - int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; - const char *data; - - std::vector arg_list; - va_list arguments; - va_start(arguments, (test_func)); - while ((data = va_arg(arguments, const char *))) { - arg_list.push_back(data); - } - - va_end(arguments); - - ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); - std::shared_ptr NN; - NN = nnmodel->network; - - returnable f = [&]() { - return NN->train((train_func), (val_func), (test_func), arg_list); - }; - - status = nntrainer_exception_boundary(f); - - return status; -} - int ml_train_model_destroy(ml_train_model_h model) { int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; + ml_train_model *nnmodel; ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); @@ -243,6 +216,8 @@ int ml_train_model_destroy(ml_train_model_h model) { if (nnmodel->optimizer) delete nnmodel->optimizer; + if (nnmodel->dataset) + delete nnmodel->dataset; for (auto &x : nnmodel->layers_map) delete (x.second); nnmodel->layers_map.clear(); @@ -253,12 +228,17 @@ int ml_train_model_destroy(ml_train_model_h model) { int ml_train_model_add_layer(ml_train_model_h model, ml_train_layer_h layer) { int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; - ml_nnlayer *nnlayer; + ml_train_model *nnmodel; + ml_train_layer *nnlayer; ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer); + if (nnlayer->in_use) { + ml_loge("Layer already in use."); + return ML_ERROR_INVALID_PARAMETER; + } + std::shared_ptr NN; std::shared_ptr NL; @@ -279,12 +259,17 @@ int ml_train_model_add_layer(ml_train_model_h model, ml_train_layer_h layer) { int ml_train_model_set_optimizer(ml_train_model_h model, ml_train_optimizer_h optimizer) { int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; - ml_nnopt *nnopt; + ml_train_model *nnmodel; + ml_train_optimizer *nnopt; ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, optimizer); + if (nnopt->in_use) { + ml_loge("Optimizer already in use."); + return ML_ERROR_INVALID_PARAMETER; + } + std::shared_ptr NN; std::shared_ptr opt; @@ -304,16 +289,49 @@ int ml_train_model_set_optimizer(ml_train_model_h model, return status; } +int ml_train_model_set_dataset(ml_train_model_h model, + ml_train_dataset_h dataset) { + int status = ML_ERROR_NONE; + ml_train_model *nnmodel; + ml_train_dataset *nndataset; + + ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); + ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset); + + if (nndataset->in_use) { + ml_loge("Dataset already in use."); + return ML_ERROR_INVALID_PARAMETER; + } + + std::shared_ptr NN; + std::shared_ptr data; + + NN = nnmodel->network; + data = nndataset->data_buffer; + + returnable f = [&]() { return NN->setDataBuffer(data); }; + + status = nntrainer_exception_boundary(f); + if (status == ML_ERROR_NONE) { + nndataset->in_use = true; + if (nnmodel->dataset) + nnmodel->dataset->in_use = false; + nnmodel->dataset = nndataset; + } + + return status; +} + int ml_train_model_get_layer(ml_train_model_h model, const char *layer_name, ml_train_layer_h *layer) { int status = ML_ERROR_NONE; - ml_nnmodel *nnmodel; + ml_train_model *nnmodel; ML_NNTRAINER_CHECK_MODEL_VALIDATION(nnmodel, model); std::shared_ptr NN; std::shared_ptr NL; - std::unordered_map::iterator layer_iter = + std::unordered_map::iterator layer_iter = nnmodel->layers_map.find(std::string(layer_name)); /** if layer found in layers_map, return layer */ if (layer_iter != nnmodel->layers_map.end()) { @@ -332,7 +350,7 @@ int ml_train_model_get_layer(ml_train_model_h model, const char *layer_name, if (status != ML_ERROR_NONE) return status; - ml_nnlayer *nnlayer = new ml_nnlayer; + ml_train_layer *nnlayer = new ml_train_layer; nnlayer->magic = ML_NNTRAINER_MAGIC; nnlayer->layer = NL; nnlayer->in_use = true; @@ -345,7 +363,7 @@ int ml_train_model_get_layer(ml_train_model_h model, const char *layer_name, int ml_train_layer_create(ml_train_layer_h *layer, ml_train_layer_type_e type) { int status = ML_ERROR_NONE; returnable f; - ml_nnlayer *nnlayer = new ml_nnlayer; + ml_train_layer *nnlayer = new ml_train_layer; nnlayer->magic = ML_NNTRAINER_MAGIC; try { @@ -375,7 +393,7 @@ int ml_train_layer_create(ml_train_layer_h *layer, ml_train_layer_type_e type) { int ml_train_layer_destroy(ml_train_layer_h layer) { int status = ML_ERROR_NONE; - ml_nnlayer *nnlayer; + ml_train_layer *nnlayer; ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer); @@ -392,7 +410,7 @@ int ml_train_layer_destroy(ml_train_layer_h layer) { int ml_train_layer_set_property(ml_train_layer_h layer, ...) { int status = ML_ERROR_NONE; - ml_nnlayer *nnlayer; + ml_train_layer *nnlayer; const char *data; ML_NNTRAINER_CHECK_LAYER_VALIDATION(nnlayer, layer); @@ -420,7 +438,7 @@ int ml_train_optimizer_create(ml_train_optimizer_h *optimizer, ml_train_optimizer_type_e type) { int status = ML_ERROR_NONE; - ml_nnopt *nnopt = new ml_nnopt; + ml_train_optimizer *nnopt = new ml_train_optimizer; nnopt->magic = ML_NNTRAINER_MAGIC; nnopt->optimizer = std::make_shared(); nnopt->in_use = false; @@ -441,7 +459,7 @@ int ml_train_optimizer_create(ml_train_optimizer_h *optimizer, int ml_train_optimizer_destroy(ml_train_optimizer_h optimizer) { int status = ML_ERROR_NONE; - ml_nnopt *nnopt; + ml_train_optimizer *nnopt; ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, optimizer); @@ -457,9 +475,9 @@ int ml_train_optimizer_destroy(ml_train_optimizer_h optimizer) { int ml_train_optimizer_set_property(ml_train_optimizer_h optimizer, ...) { int status = ML_ERROR_NONE; - ml_nnopt *nnopt; + ml_train_optimizer *nnopt; const char *data; - nnopt = (ml_nnopt *)optimizer; + nnopt = (ml_train_optimizer *)optimizer; ML_NNTRAINER_CHECK_OPT_VALIDATION(nnopt, optimizer); std::vector arg_list; @@ -483,6 +501,122 @@ int ml_train_optimizer_set_property(ml_train_optimizer_h optimizer, ...) { return status; } +int ml_train_dataset_create_with_generator(ml_train_dataset_h *dataset, + ml_train_datagen_cb train_cb, + ml_train_datagen_cb valid_cb, + ml_train_datagen_cb test_cb) { + int status = ML_ERROR_NONE; + + std::shared_ptr data_buffer = + std::make_shared(); + + status = data_buffer->setFunc(nntrainer::BUF_TRAIN, train_cb); + if (status != ML_ERROR_NONE) { + return status; + } + + status = data_buffer->setFunc(nntrainer::BUF_VAL, valid_cb); + if (status != ML_ERROR_NONE) { + return status; + } + + status = data_buffer->setFunc(nntrainer::BUF_TEST, test_cb); + if (status != ML_ERROR_NONE) { + return status; + } + + ml_train_dataset *nndataset = new ml_train_dataset; + nndataset->magic = ML_NNTRAINER_MAGIC; + nndataset->data_buffer = data_buffer; + nndataset->in_use = false; + + *dataset = nndataset; + return status; +} + +int ml_train_dataset_create_with_file(ml_train_dataset_h *dataset, + const char *train_file, + const char *valid_file, + const char *test_file) { + int status = ML_ERROR_NONE; + + std::shared_ptr data_buffer = + std::make_shared(); + + if (train_file) { + status = data_buffer->setDataFile(train_file, nntrainer::DATA_TRAIN); + if (status != ML_ERROR_NONE) { + return status; + } + } else { + ml_loge("Train data file must be valid."); + return ML_ERROR_INVALID_PARAMETER; + } + + if (valid_file) { + status = data_buffer->setDataFile(valid_file, nntrainer::DATA_VAL); + if (status != ML_ERROR_NONE) { + return status; + } + } + + if (test_file) { + status = data_buffer->setDataFile(test_file, nntrainer::DATA_TEST); + if (status != ML_ERROR_NONE) { + return status; + } + } + + ml_train_dataset *nndataset = new ml_train_dataset; + nndataset->magic = ML_NNTRAINER_MAGIC; + nndataset->data_buffer = data_buffer; + nndataset->in_use = false; + + *dataset = nndataset; + return status; +} + +int ml_train_dataset_set_property(ml_train_dataset_h dataset, ...) { + int status = ML_ERROR_NONE; + ml_train_dataset *nndataset; + const char *data; + + ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset); + + std::vector arg_list; + va_list arguments; + va_start(arguments, dataset); + + while ((data = va_arg(arguments, const char *))) { + arg_list.push_back(data); + } + + va_end(arguments); + + returnable f = [&]() { + return nndataset->data_buffer->setProperty(arg_list); + }; + status = nntrainer_exception_boundary(f); + + return status; +} + +int ml_train_dataset_destroy(ml_train_dataset_h dataset) { + int status = ML_ERROR_NONE; + ml_train_dataset *nndataset; + + ML_NNTRAINER_CHECK_DATASET_VALIDATION(nndataset, dataset); + + if (nndataset->in_use) { + ml_loge("Cannot delete dataset already set to a model." + "Delete model will delete this dataset."); + return ML_ERROR_INVALID_PARAMETER; + } + + delete nndataset; + return status; +} + #ifdef __cplusplus } #endif diff --git a/jni/Android.mk b/jni/Android.mk index 71bd063..587d5a5 100644 --- a/jni/Android.mk +++ b/jni/Android.mk @@ -41,7 +41,8 @@ NNTRAINER_SRCS := $(NNTRAINER_ROOT)/nntrainer/src/neuralnet.cpp \ $(NNTRAINER_ROOT)/nntrainer/src/flatten_layer.cpp NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer/include \ - $(NNTRAINER_ROOT)/api/capi/include/platform + $(NNTRAINER_ROOT)/api/capi/include \ + $(NNTRAINER_ROOT)/api/capi/include/platform INIPARSER_SRCS := $(INIPARSER_ROOT)/src/iniparser.c \ $(INIPARSER_ROOT)/src/dictionary.c diff --git a/meson.build b/meson.build index 6ae82df..04ee03f 100644 --- a/meson.build +++ b/meson.build @@ -42,7 +42,7 @@ warning_c_flags = [ '-Waggregate-return', '-Wold-style-definition', '-Wdeclaration-after-statement', - '-Wno-error=varargs' + '-Wno-error=varargs' ] foreach extra_arg : warning_flags diff --git a/nntrainer/include/databuffer.h b/nntrainer/include/databuffer.h index 9cfc608..d71689c 100644 --- a/nntrainer/include/databuffer.h +++ b/nntrainer/include/databuffer.h @@ -36,7 +36,7 @@ #include #include -/* +/** * @brief Number of Data Set */ #define NBUFTYPE 4 @@ -107,6 +107,18 @@ typedef enum { } DataType; /** + * @brief Enumeration for data buffer type + * 0. DATA_BUFFER_GENERATOR + * 1. DATA_BUFFER_FILE + * 2. DATA_BUFFER_UNKNOWN + */ +typedef enum { + DATA_BUFFER_GENERATOR, + DATA_BUFFER_FILE, + DATA_BUFFER_UNKNOWN +} DataBufferType; + +/** * @class DataBuffer Data Buffers * @brief Data Buffer for read and manage data */ @@ -116,13 +128,14 @@ public: * @brief Create Buffer * @retval DataBuffer */ - DataBuffer() : + DataBuffer(DataBufferType type) : train_running(), val_running(), test_running(), train_thread(), val_thread(), - test_thread() { + test_thread(), + data_buffer_type(type) { SET_VALIDATION(false); class_num = 0; cur_train_bufsize = 0; @@ -264,12 +277,29 @@ public: bool *getValidation() { return validation; } /** + * @brief set property + * @param[in] values values of property + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + */ + int setProperty(std::vector values); + + /** * @brief status of thread */ DataStatus trainReadyFlag; DataStatus valReadyFlag; DataStatus testReadyFlag; + enum class PropertyType { + train_data = 0, + val_data = 1, + test_data = 2, + label_data = 3, + buffer_size = 4, + unknown = 5 + }; + protected: /** * @brief Data Queues for each data set @@ -354,6 +384,21 @@ protected: int rangeRandom(int min, int max); std::mt19937 rng; + + /** + * @brief The type of data buffer + */ + DataBufferType data_buffer_type; + + /** + * @brief set property + * @param[in] type type of property + * @param[in] value string value of property + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + */ + virtual int setProperty(const DataBuffer::PropertyType type, + std::string &value); }; } // namespace nntrainer diff --git a/nntrainer/include/databuffer_file.h b/nntrainer/include/databuffer_file.h index 0417243..45a7340 100644 --- a/nntrainer/include/databuffer_file.h +++ b/nntrainer/include/databuffer_file.h @@ -46,7 +46,7 @@ public: /** * @brief Constructor */ - DataBufferFromDataFile(){}; + DataBufferFromDataFile() : DataBuffer(DataBufferType::DATA_BUFFER_FILE){}; /** * @brief Destructor @@ -84,6 +84,15 @@ public: */ int setFeatureSize(TensorDim indim); + /** + * @brief set property + * @param[in] type type of property + * @param[in] value string value of property + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + */ + int setProperty(const PropertyType type, std::string &value); + private: /** * @brief raw data file names diff --git a/nntrainer/include/databuffer_func.h b/nntrainer/include/databuffer_func.h index bb5b685..8f99d6a 100644 --- a/nntrainer/include/databuffer_func.h +++ b/nntrainer/include/databuffer_func.h @@ -31,12 +31,19 @@ #include #include #include +#include #include #include namespace nntrainer { /** + * @brief Dataset generator callback type declaration + */ +typedef std::function::type> + datagen_cb; + +/** * @class DataBufferFromCallback Data Buffer from callback given by user * @brief Data Buffer from callback function */ @@ -45,7 +52,8 @@ public: /** * @brief Constructor */ - DataBufferFromCallback(){}; + DataBufferFromCallback() : + DataBuffer(DataBufferType::DATA_BUFFER_GENERATOR){}; /** * @brief Destructor @@ -66,8 +74,7 @@ public: * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - int setFunc(BufferType type, - std::function func); + int setFunc(BufferType type, datagen_cb func); /** * @brief Update Data Buffer ( it is for child thread ) @@ -76,6 +83,15 @@ public: */ void updateData(BufferType type); + /** + * @brief set property + * @param[in] type type of property + * @param[in] value string value of property + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + */ + int setProperty(const PropertyType type, std::string &value); + private: /** * @@ -86,9 +102,9 @@ private: * @retval true / false generate all data for this epoch * */ - std::function callback_train; - std::function callback_val; - std::function callback_test; + datagen_cb callback_train; + datagen_cb callback_val; + datagen_cb callback_test; }; } // namespace nntrainer #endif /* __cplusplus */ diff --git a/nntrainer/include/neuralnet.h b/nntrainer/include/neuralnet.h index e50c12f..9626240 100644 --- a/nntrainer/include/neuralnet.h +++ b/nntrainer/include/neuralnet.h @@ -221,29 +221,11 @@ public: * mini batch size data per every call. * @param[in] test_func callback function to get test data. This provides * mini batch size data per every call. - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. - */ - int train(std::function function_train, - std::function function_val, - std::function function_test); - - /** - * @brief Run NeuralNetwork train with callback function by user - * @param[in] train_func callback function to get train data. This provides - * mini batch size data per every call. - * @param[in] val_func callback function to get validation data. This provides - * mini batch size data per every call. - * @param[in] test_func callback function to get test data. This provides - * mini batch size data per every call. * @param[in] values hyper-parameter list * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - int train(std::function function_train, - std::function function_val, - std::function function_test, - std::vector values); + int setDataBuffer(std::shared_ptr data_buffer); /** * @brief check neural network whether the hyper-parameters are set. @@ -286,16 +268,11 @@ public: enum class PropertyType { loss = 0, cost = 1, - train_data = 2, - val_data = 3, - test_data = 4, - label_data = 5, - buffer_size = 6, - batch_size = 7, - epochs = 8, - model_file = 9, - continue_train = 10, - unknown = 11, + batch_size = 2, + epochs = 3, + model_file = 4, + continue_train = 5, + unknown = 6 }; private: diff --git a/nntrainer/include/parse_util.h b/nntrainer/include/parse_util.h index de10d35..6928f6e 100644 --- a/nntrainer/include/parse_util.h +++ b/nntrainer/include/parse_util.h @@ -117,6 +117,13 @@ unsigned int parseOptProperty(std::string property); unsigned int parseNetProperty(std::string property); /** + * @brief Parsing Data Buffer Property + * @param[in] property string to be parsed + * @retval int enumerated type + */ +unsigned int parseDataProperty(std::string property); + +/** * @brief check str to be int and assign * @param[out] val assign variable * @param[in] str input string diff --git a/nntrainer/meson.build b/nntrainer/meson.build index ee4722a..3245b45 100644 --- a/nntrainer/meson.build +++ b/nntrainer/meson.build @@ -1,5 +1,6 @@ nntrainer_inc = [ - include_directories('./include') + include_directories('./include'), + include_directories('../api/capi/include') ] # pc file is not present for 'ml-api-common' yet diff --git a/nntrainer/src/bn_layer.cpp b/nntrainer/src/bn_layer.cpp index 7be9e17..8b0e026 100644 --- a/nntrainer/src/bn_layer.cpp +++ b/nntrainer/src/bn_layer.cpp @@ -64,7 +64,6 @@ int BatchNormalizationLayer::initialize(bool last) { int BatchNormalizationLayer::setOptimizer(Optimizer &opt) { this->opt.setType(opt.getType()); this->opt.setOptParam(opt.getOptParam()); - this->epsilon = epsilon; return this->opt.initialize(dim, false); } diff --git a/nntrainer/src/databuffer.cpp b/nntrainer/src/databuffer.cpp index 5a86cb8..aa9d3de 100644 --- a/nntrainer/src/databuffer.cpp +++ b/nntrainer/src/databuffer.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -401,4 +402,44 @@ void DataBuffer::displayProgress(const int count, BufferType type, float loss) { std::cout.flush(); } +int DataBuffer::setProperty(std::vector values) { + int status = ML_ERROR_NONE; + + for (unsigned int i = 0; i < values.size(); ++i) { + std::string key; + std::string value; + status = getKeyValue(values[i], key, value); + NN_RETURN_STATUS(); + + unsigned int type = parseDataProperty(key); + if (value.empty()) + return ML_ERROR_INVALID_PARAMETER; + + status = setProperty(static_cast(type), value); + NN_RETURN_STATUS(); + } + + return status; +} + +int DataBuffer::setProperty(const PropertyType type, std::string &value) { + int status = ML_ERROR_NONE; + + switch (type) { + case PropertyType::buffer_size: + int size; + status = setInt(size, value); + NN_RETURN_STATUS(); + status = this->setBufSize(size); + NN_RETURN_STATUS(); + break; + default: + ml_loge("Error: Unknown Data Buffer Property Key"); + status = ML_ERROR_INVALID_PARAMETER; + break; + } + + return status; +} + } /* namespace nntrainer */ diff --git a/nntrainer/src/databuffer_file.cpp b/nntrainer/src/databuffer_file.cpp index 3b81e34..a04707f 100644 --- a/nntrainer/src/databuffer_file.cpp +++ b/nntrainer/src/databuffer_file.cpp @@ -381,4 +381,32 @@ int DataBufferFromDataFile::setFeatureSize(TensorDim tdim) { return status; } +int DataBufferFromDataFile::setProperty(const PropertyType type, + std::string &value) { + int status = ML_ERROR_NONE; + + if (data_buffer_type != DATA_BUFFER_FILE) + return ML_ERROR_INVALID_PARAMETER; + + switch (type) { + case PropertyType::train_data: + status = this->setDataFile(value, DATA_TRAIN); + break; + case PropertyType::val_data: + status = this->setDataFile(value, DATA_VAL); + break; + case PropertyType::test_data: + status = this->setDataFile(value, DATA_TEST); + break; + case PropertyType::label_data: + status = this->setDataFile(value, DATA_LABEL); + break; + default: + status = DataBuffer::setProperty(type, value); + break; + } + + return status; +} + } /* namespace nntrainer */ diff --git a/nntrainer/src/databuffer_func.cpp b/nntrainer/src/databuffer_func.cpp index 773b647..41a802a 100644 --- a/nntrainer/src/databuffer_func.cpp +++ b/nntrainer/src/databuffer_func.cpp @@ -107,8 +107,7 @@ int DataBufferFromCallback::init() { return status; } -int DataBufferFromCallback::setFunc( - BufferType type, std::function func) { +int DataBufferFromCallback::setFunc(BufferType type, datagen_cb func) { int status = ML_ERROR_NONE; switch (type) { @@ -143,7 +142,7 @@ void DataBufferFromCallback::updateData(BufferType type) { bool *running = NULL; std::vector> *data = NULL; std::vector> *datalabel = NULL; - std::function callback; + datagen_cb callback; switch (type) { case BUF_TRAIN: { @@ -185,20 +184,27 @@ void DataBufferFromCallback::updateData(BufferType type) { } bool endflag = false; + float **vec_arr = (float **)malloc(sizeof(float *) * 1); + float **veclabel_arr = (float **)malloc(sizeof(float *) * 1); + float *vec = (float *)malloc(sizeof(float) * input_dim.batch() * input_dim.channel() * input_dim.height() * input_dim.width()); float *veclabel = (float *)malloc(sizeof(float) * input_dim.batch() * class_num); + vec_arr[0] = vec; + veclabel_arr[0] = veclabel; + while ((*running)) { trainReadyFlag = DATA_NOT_READY; valReadyFlag = DATA_NOT_READY; testReadyFlag = DATA_NOT_READY; if (buf_size - (*cur_size) > 0) { - endflag = callback(vec, veclabel, &status); + /** @todo Update to support multiple inputs later */ + status = callback(vec_arr, veclabel_arr, &endflag); - if (endflag) { + if (status == ML_ERROR_NONE && !endflag) { for (unsigned int i = 0; i < input_dim.batch(); ++i) { std::vector v; std::vector vl; @@ -227,15 +233,25 @@ void DataBufferFromCallback::updateData(BufferType type) { } } - if (buf_size == (*cur_size) || !endflag) { + if (buf_size == (*cur_size) || endflag) { switch (type) { case BUF_TRAIN: { std::lock_guard lgtrain(readyTrainData); - if (!endflag) { + if (status != ML_ERROR_NONE) { + trainReadyFlag = DATA_ERROR; + cv_train.notify_all(); + free(vec); + free(veclabel); + free(vec_arr); + free(veclabel_arr); + return; + } else if (endflag) { trainReadyFlag = DATA_END; cv_train.notify_all(); free(vec); free(veclabel); + free(vec_arr); + free(veclabel_arr); return; } else { trainReadyFlag = DATA_READY; @@ -245,11 +261,21 @@ void DataBufferFromCallback::updateData(BufferType type) { } break; case BUF_VAL: { std::lock_guard lgval(readyValData); - if (!endflag) { + if (status != ML_ERROR_NONE) { + valReadyFlag = DATA_ERROR; + cv_val.notify_all(); + free(vec); + free(veclabel); + free(vec_arr); + free(veclabel_arr); + return; + } else if (endflag) { valReadyFlag = DATA_END; - cv_train.notify_all(); + cv_val.notify_all(); free(vec); free(veclabel); + free(vec_arr); + free(veclabel_arr); return; } else { valReadyFlag = DATA_READY; @@ -259,11 +285,21 @@ void DataBufferFromCallback::updateData(BufferType type) { } break; case BUF_TEST: { std::lock_guard lgtest(readyTestData); - if (!endflag) { + if (status != ML_ERROR_NONE) { + testReadyFlag = DATA_ERROR; + cv_test.notify_all(); + free(vec); + free(veclabel); + free(vec_arr); + free(veclabel_arr); + return; + } else if (endflag) { testReadyFlag = DATA_END; cv_test.notify_all(); free(vec); free(veclabel); + free(vec_arr); + free(veclabel_arr); return; } else { testReadyFlag = DATA_READY; @@ -280,4 +316,9 @@ void DataBufferFromCallback::updateData(BufferType type) { free(veclabel); } +int DataBufferFromCallback::setProperty(const PropertyType type, + std::string &value) { + return DataBuffer::setProperty(type, value); +} + } /* namespace nntrainer */ diff --git a/nntrainer/src/neuralnet.cpp b/nntrainer/src/neuralnet.cpp index a1dc3a9..23b8faf 100644 --- a/nntrainer/src/neuralnet.cpp +++ b/nntrainer/src/neuralnet.cpp @@ -567,6 +567,7 @@ int NeuralNetwork::setTrainConfig(std::vector values) { unsigned int type = parseNetProperty(key); + /** TODO: disable this batch size */ switch (static_cast(type)) { case PropertyType::batch_size: { status = setInt(batch_size, value); @@ -586,34 +587,6 @@ int NeuralNetwork::setTrainConfig(std::vector values) { NN_RETURN_STATUS(); epoch = e; } break; - case PropertyType::train_data: { - status = std::static_pointer_cast(data_buffer) - ->setDataFile(value, DATA_TRAIN); - NN_RETURN_STATUS(); - } break; - case PropertyType::val_data: { - status = std::static_pointer_cast(data_buffer) - ->setDataFile(value, DATA_VAL); - NN_RETURN_STATUS(); - } break; - case PropertyType::test_data: { - status = std::static_pointer_cast(data_buffer) - ->setDataFile(value, DATA_TEST); - NN_RETURN_STATUS(); - } break; - case PropertyType::label_data: { - status = std::static_pointer_cast(data_buffer) - ->setDataFile(value, DATA_LABEL); - NN_RETURN_STATUS(); - } break; - - case PropertyType::buffer_size: { - int size; - status = setInt(size, value); - NN_RETURN_STATUS(); - status = data_buffer->setBufSize(size); - NN_RETURN_STATUS(); - } break; case PropertyType::model_file: { model = value; } break; @@ -851,64 +824,13 @@ int NeuralNetwork::train(std::vector values) { int status = ML_ERROR_NONE; if (data_buffer == nullptr) { - data_buffer = std::make_shared(); - } - - if (values.size() != 0) { - status = data_buffer->setMiniBatch(layers[0]->getInputDimension().batch()); - NN_RETURN_STATUS(); - - status = setTrainConfig(values); - NN_RETURN_STATUS(); + ml_loge("Cannot initialize the model without the data buffer."); + return ML_ERROR_INVALID_PARAMETER; } - status = data_buffer->setClassNum( - layers[layers.size() - 1]->getOutputDimension().width()); - NN_RETURN_STATUS(); - - status = data_buffer->setFeatureSize(layers[0]->getInputDimension()); - NN_RETURN_STATUS(); - - status = data_buffer->init(); + status = data_buffer->setMiniBatch(layers[0]->getInputDimension().batch()); NN_RETURN_STATUS(); - return train_run(); -} - -/** - * @brief Run NeuralNetwork train - */ -int NeuralNetwork::train( - std::function train_func, - std::function val_func, - std::function test_func) { - - std::vector values; - - return train(train_func, val_func, test_func, values); -} - -/** - * @brief Run NeuralNetwork train - */ -int NeuralNetwork::train( - std::function train_func, - std::function val_func, - std::function test_func, - std::vector values) { - - int status = ML_ERROR_NONE; - - if (data_buffer == nullptr) { - data_buffer = std::make_shared(); - - status = data_buffer->setMiniBatch(layers[0]->getInputDimension().batch()); - NN_RETURN_STATUS(); - - status = setTrainConfig(values); - NN_RETURN_STATUS(); - } - status = data_buffer->setClassNum( layers[layers.size() - 1]->getOutputDimension().width()); NN_RETURN_STATUS(); @@ -919,23 +841,11 @@ int NeuralNetwork::train( status = data_buffer->init(); NN_RETURN_STATUS(); - std::shared_ptr callback_buffer = - std::static_pointer_cast(data_buffer); - - status = callback_buffer->setFunc(nntrainer::BUF_TRAIN, (train_func)); - if (status != ML_ERROR_NONE) - return status; - - status = callback_buffer->setFunc(nntrainer::BUF_VAL, (val_func)); - if (status != ML_ERROR_NONE) - return status; - - status = callback_buffer->setFunc(nntrainer::BUF_TEST, (test_func)); - if (status != ML_ERROR_NONE) - return status; + status = setTrainConfig(values); + NN_RETURN_STATUS(); return train_run(); -}; +} /** * @brief Run NeuralNetwork train with callback function by user @@ -1088,6 +998,12 @@ int NeuralNetwork::setOptimizer(std::shared_ptr optimizer) { return ML_ERROR_NONE; } +int NeuralNetwork::setDataBuffer(std::shared_ptr data_buffer) { + this->data_buffer = data_buffer; + + return ML_ERROR_NONE; +} + void NeuralNetwork::ensureName(std::shared_ptr layer, std::string prefix) { if (layer->getName().empty()) { diff --git a/nntrainer/src/parse_util.cpp b/nntrainer/src/parse_util.cpp index eeec49a..1aa7a34 100644 --- a/nntrainer/src/parse_util.cpp +++ b/nntrainer/src/parse_util.cpp @@ -333,18 +333,12 @@ unsigned int parseNetProperty(std::string property) { * @brief Network Properties * loss = 0, * cost = 1, - * train_data = 2, - * val_data = 3, - * test_data = 4, - * label_data = 5, - * buffer_size = 6, - * batch_size = 7, - * epochs = 8, - * model_file = 9 + * batch_size = 2, + * epochs = 3, + * model_file = 4 */ - std::array property_string = { - "loss", "cost", "train_data", "val_data", "test_data", - "label_data", "buffer_size", "batch_size", "epochs", "model_file"}; + std::array property_string = {"loss", "cost", "batch_size", + "epochs", "model_file"}; for (i = 0; i < property_string.size(); i++) { unsigned int size = (property_string[i].size() > property.size()) @@ -359,6 +353,33 @@ unsigned int parseNetProperty(std::string property) { return (unsigned int)NeuralNetwork::PropertyType::unknown; } +unsigned int parseDataProperty(std::string property) { + unsigned int i; + + /** + * @brief Network Properties + * train_data = 0, + * val_data = 1, + * test_data = 2, + * label_data = 3, + * buffer_size = 4 + */ + std::array property_string = { + "train_data", "val_data", "test_data", "label_data", "buffer_size"}; + + for (i = 0; i < property_string.size(); i++) { + unsigned int size = (property_string[i].size() > property.size()) + ? property_string[i].size() + : property.size(); + + if (!strncasecmp(property_string[i].c_str(), property.c_str(), size)) { + return (i); + } + } + + return (unsigned int)DataBuffer::PropertyType::unknown; +} + int setInt(int &val, std::string str) { int status = ML_ERROR_NONE; try { diff --git a/test/include/nntrainer_test_util.h b/test/include/nntrainer_test_util.h index ed5888d..6f579bb 100644 --- a/test/include/nntrainer_test_util.h +++ b/test/include/nntrainer_test_util.h @@ -231,19 +231,19 @@ void replaceString(const std::string &from, const std::string &to, * @brief get data which size is mini batch for train * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_train(float *outVec, float *outLabel, int *status); +int getMiniBatch_train(float **outVec, float **outLabel, bool *last); /** * @brief get data which size is mini batch for val * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_val(float *outVec, float *outLabel, int *status); +int getMiniBatch_val(float **outVec, float **outLabel, bool *last); #endif /* __cplusplus */ #endif /* __NNTRAINER_TEST_UTIL_H__ */ diff --git a/test/nntrainer_test_util.cpp b/test/nntrainer_test_util.cpp index d3ff023..7f1cda4 100644 --- a/test/nntrainer_test_util.cpp +++ b/test/nntrainer_test_util.cpp @@ -24,6 +24,7 @@ #include "nntrainer_test_util.h" #include #include +#include #include #include @@ -110,14 +111,15 @@ static bool getData(std::ifstream &F, std::vector &outVec, * @brief get data which size is mini batch for train * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { +int getMiniBatch_train(float **outVec, float **outLabel, bool *last) { std::vector memI; std::vector memJ; unsigned int count = 0; unsigned int data_size = 0; + *last = true; std::string filename = "trainingSet.dat"; std::ifstream F(filename, std::ios::in | std::ios::binary); @@ -145,7 +147,8 @@ bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { if (count < mini_batch) { free(duplicate); alloc_train = false; - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -168,28 +171,30 @@ bool getMiniBatch_train(float *outVec, float *outLabel, int *status) { getData(F, o, l, memI[i]); for (unsigned int j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (unsigned int j = 0; j < num_class; ++j) - outLabel[i * num_class + j] = l[j]; + outLabel[0][i * num_class + j] = l[j]; } F.close(); - return true; + *last = false; + return ML_ERROR_NONE; } /** * @brief get data which size is mini batch for validation * @param[out] outVec * @param[out] outLabel - * @param[out] status for error handling - * @retval true/false false : end of data + * @param[out] last if the data is finished + * @retval status for handling error */ -bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { +int getMiniBatch_val(float **outVec, float **outLabel, bool *last) { std::vector memI; std::vector memJ; unsigned int count = 0; unsigned int data_size = 0; + *last = true; std::string filename = "trainingSet.dat"; std::ifstream F(filename, std::ios::in | std::ios::binary); @@ -217,7 +222,8 @@ bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { if (count < mini_batch) { free(valduplicate); alloc_val = false; - return false; + *last = true; + return ML_ERROR_NONE; } count = 0; @@ -240,13 +246,14 @@ bool getMiniBatch_val(float *outVec, float *outLabel, int *status) { getData(F, o, l, memI[i]); for (unsigned int j = 0; j < feature_size; ++j) - outVec[i * feature_size + j] = o[j]; + outVec[0][i * feature_size + j] = o[j]; for (unsigned int j = 0; j < num_class; ++j) - outLabel[i * num_class + j] = l[j]; + outLabel[0][i * num_class + j] = l[j]; } F.close(); - return true; + *last = false; + return ML_ERROR_NONE; } /** diff --git a/test/tizen_capi/unittest_tizen_capi.cpp b/test/tizen_capi/unittest_tizen_capi.cpp index dc32518..49ad707 100644 --- a/test/tizen_capi/unittest_tizen_capi.cpp +++ b/test/tizen_capi/unittest_tizen_capi.cpp @@ -259,7 +259,7 @@ TEST(nntrainer_capi_nnmodel, train_01_p) { int status = ML_ERROR_NONE; std::string config_file = "./test_train_01_p.ini"; RESET_CONFIG(config_file.c_str()); - replaceString("Input_Shape = 32:1:1:62720", "Input_Shape=32:1:1:62720", + replaceString("Input_Shape = 32:1:1:62720", "Input_Shape=16:1:1:62720", config_file, config_str); replaceString("minibatch = 32", "minibatch = 16", config_file, config_str); replaceString("BufferSize=100", "", config_file, config_str); @@ -267,7 +267,7 @@ TEST(nntrainer_capi_nnmodel, train_01_p) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_compile(handle, NULL); EXPECT_EQ(status, ML_ERROR_NONE); - status = ml_nnmodel_train_with_file(handle, NULL); + status = ml_train_model_run(handle, NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_destroy(handle); EXPECT_EQ(status, ML_ERROR_NONE); @@ -278,7 +278,7 @@ TEST(nntrainer_capi_nnmodel, train_01_p) { */ TEST(nntrainer_capi_nnmodel, train_02_n) { int status = ML_ERROR_NONE; - status = ml_nnmodel_train_with_file(NULL, NULL); + status = ml_train_model_run(NULL, NULL); EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER); } @@ -298,7 +298,7 @@ TEST(nntrainer_capi_nnmodel, train_03_n) { EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_compile(handle, NULL); EXPECT_EQ(status, ML_ERROR_NONE); - status = ml_nnmodel_train_with_file(handle, "loss=cross", NULL); + status = ml_train_model_run(handle, "loss=cross", NULL); EXPECT_EQ(status, ML_ERROR_INVALID_PARAMETER); status = ml_train_model_destroy(handle); EXPECT_EQ(status, ML_ERROR_NONE); @@ -608,6 +608,7 @@ TEST(nntrainer_capi_nnmodel, train_with_file_01_p) { ml_train_model_h model; ml_train_layer_h layers[2]; ml_train_optimizer_h optimizer; + ml_train_dataset_h dataset; status = ml_train_model_construct(&model); EXPECT_EQ(status, ML_ERROR_NONE); @@ -646,13 +647,22 @@ TEST(nntrainer_capi_nnmodel, train_with_file_01_p) { status = ml_train_model_set_optimizer(model, optimizer); EXPECT_EQ(status, ML_ERROR_NONE); + status = ml_train_dataset_create_with_file(&dataset, "trainingSet.dat", + "valSet.dat", NULL); + EXPECT_EQ(status, ML_ERROR_NONE); + + status = ml_train_dataset_set_property(dataset, "label_data=label.dat", + "buffer_size=100", NULL); + EXPECT_EQ(status, ML_ERROR_NONE); + + status = ml_train_model_set_dataset(model, dataset); + EXPECT_EQ(status, ML_ERROR_NONE); + status = ml_train_model_compile(model, "loss=cross", NULL); EXPECT_EQ(status, ML_ERROR_NONE); - status = ml_nnmodel_train_with_file( - model, "epochs=2", "batch_size=16", "train_data=trainingSet.dat", - "val_data=valSet.dat", "label_data=label.dat", "buffer_size=100", - "model_file=model.bin", NULL); + status = ml_train_model_run(model, "epochs=2", "batch_size=16", + "model_file=model.bin", NULL); EXPECT_EQ(status, ML_ERROR_NONE); status = ml_train_model_destroy(model); @@ -668,6 +678,7 @@ TEST(nntrainer_capi_nnmodel, train_with_generator_01_p) { ml_train_model_h model; ml_train_layer_h layers[2]; ml_train_optimizer_h optimizer; + ml_train_dataset_h dataset; status = ml_train_model_construct(&model); EXPECT_EQ(status, ML_ERROR_NONE); @@ -706,12 +717,21 @@ TEST(nntrainer_capi_nnmodel, train_with_generator_01_p) { status = ml_train_model_set_optimizer(model, optimizer); EXPECT_EQ(status, ML_ERROR_NONE); + status = ml_train_dataset_create_with_generator(&dataset, getMiniBatch_train, + getMiniBatch_val, NULL); + EXPECT_EQ(status, ML_ERROR_NONE); + + status = ml_train_dataset_set_property(dataset, "buffer_size=100", NULL); + EXPECT_EQ(status, ML_ERROR_NONE); + + status = ml_train_model_set_dataset(model, dataset); + EXPECT_EQ(status, ML_ERROR_NONE); + status = ml_train_model_compile(model, "loss=cross", NULL); EXPECT_EQ(status, ML_ERROR_NONE); - status = ml_nnmodel_train_with_generator( - model, getMiniBatch_train, getMiniBatch_val, NULL, "epochs=2", - "batch_size=16", "buffer_size=100", "model_file=model.bin", NULL); + status = ml_train_model_run(model, "epochs=2", "batch_size=16", + "model_file=model.bin", NULL); status = ml_train_model_destroy(model); EXPECT_EQ(status, ML_ERROR_NONE); -- 2.7.4