From 451a5297e6b0a898bd66681ed4ced7dacfb3a179 Mon Sep 17 00:00:00 2001 From: Jihoon Lee Date: Wed, 14 Jul 2021 18:58:22 +0900 Subject: [PATCH] [Dataset] Change factory signature Add properties to factory, given contructor argument, it is delegated to the constructor **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Cc: Parichay Kapoor Signed-off-by: Jihoon Lee --- api/ccapi/include/dataset.h | 32 +++++++++++++++------ api/ccapi/src/factory.cpp | 24 ++++++++++------ nntrainer/dataset/databuffer_factory.cpp | 48 ++++++++++++++++---------------- nntrainer/dataset/databuffer_file.h | 9 ++++++ nntrainer/dataset/databuffer_func.h | 9 ++++++ nntrainer/models/model_loader.cpp | 21 ++++++-------- test/ccapi/unittest_ccapi.cpp | 6 ++-- 7 files changed, 93 insertions(+), 56 deletions(-) diff --git a/api/ccapi/include/dataset.h b/api/ccapi/include/dataset.h index 8a744c4..8799cf3 100644 --- a/api/ccapi/include/dataset.h +++ b/api/ccapi/include/dataset.h @@ -1,4 +1,3 @@ - // SPDX-License-Identifier: Apache-2.0 /** * Copyright (C) 2020 Parichay Kapoor @@ -87,23 +86,40 @@ public: }; /** - * @brief Factory creator with constructor for dataset + * @brief Create a Dataset object with given arguements + * + * @param type dataset type + * @param properties property representations + * @return std::unique_ptr created dataset */ std::unique_ptr createDataset(DatasetType type, const std::vector &properties = {}); /** - * @brief Factory creator with constructor for dataset + * @brief Create a Dataset object + * + * @param type dataset type + * @param path path to a file or folder + * @param properties property representations + * @return std::unique_ptr created dataset */ -std::unique_ptr createDataset(DatasetType type, const char *file); +std::unique_ptr +createDataset(DatasetType type, const char *path, + const std::vector &properties = {}); /** - * @brief Factory creator with constructor for dataset + * @brief Create a Dataset object + * + * @param type dataset type + * @param cb callback + * @param user_data user data + * @param properties property representations + * @return std::unique_ptr created dataset */ -std::unique_ptr createDataset(DatasetType type, datagen_cb cb, - void *user_data = nullptr); - +std::unique_ptr +createDataset(DatasetType type, datagen_cb cb, void *user_data = nullptr, + const std::vector &properties = {}); } // namespace train } // namespace ml diff --git a/api/ccapi/src/factory.cpp b/api/ccapi/src/factory.cpp index 53db60f..d30800c 100644 --- a/api/ccapi/src/factory.cpp +++ b/api/ccapi/src/factory.cpp @@ -82,25 +82,31 @@ std::unique_ptr createModel(ModelType type, std::unique_ptr createDataset(DatasetType type, const std::vector &properties) { std::unique_ptr dataset = nntrainer::createDataBuffer(type); - dataset->setProperty(properties); return dataset; } -/** - * @brief Factory creator with constructor for dataset - */ -std::unique_ptr createDataset(DatasetType type, const char *file) { - return nntrainer::createDataBuffer(type, file); +std::unique_ptr +createDataset(DatasetType type, const char *file, + const std::vector &properties) { + std::unique_ptr dataset = nntrainer::createDataBuffer(type, file); + dataset->setProperty(properties); + + return dataset; } /** * @brief Factory creator with constructor for dataset */ -std::unique_ptr createDataset(DatasetType type, datagen_cb cb, - void *user_data) { - return nntrainer::createDataBuffer(type, cb, user_data); +std::unique_ptr +createDataset(DatasetType type, datagen_cb cb, void *user_data, + const std::vector &properties) { + std::unique_ptr dataset = + nntrainer::createDataBuffer(type, cb, user_data); + dataset->setProperty(properties); + + return dataset; } } // namespace train diff --git a/nntrainer/dataset/databuffer_factory.cpp b/nntrainer/dataset/databuffer_factory.cpp index 07c2b76..bbd12e5 100644 --- a/nntrainer/dataset/databuffer_factory.cpp +++ b/nntrainer/dataset/databuffer_factory.cpp @@ -23,14 +23,12 @@ namespace nntrainer { */ std::unique_ptr createDataBuffer(DatasetType type) { switch (type) { - case DatasetType::GENERATOR: - return std::make_unique(); case DatasetType::FILE: return std::make_unique(); case DatasetType::UNKNOWN: - /** fallthrough intended */ + [[fallthrough]]; default: - throw std::invalid_argument("Unknown type for the dataset"); + throw std::invalid_argument("Unsupported constructor type for the dataset"); } } @@ -39,17 +37,19 @@ std::unique_ptr createDataBuffer(DatasetType type) { */ std::unique_ptr createDataBuffer(DatasetType type, const char *file) { - if (type != DatasetType::FILE) - throw std::invalid_argument( - "Cannot create dataset with files with the given dataset type"); - - std::unique_ptr dataset = createDataBuffer(type); + NNTR_THROW_IF(file == nullptr, std::invalid_argument) + << "file shall not be null, use empty constructor instead"; - NNTR_THROW_IF(file == nullptr || dataset->setDataFile(file) != ML_ERROR_NONE, - std::invalid_argument) - << "invalid train file, path: " << (file ? file : "null"); - - return dataset; + switch (type) { + case DatasetType::FILE: + return std::make_unique(file); + case DatasetType::UNKNOWN: + [[fallthrough]]; + default: + throw std::invalid_argument( + "Unsupported constructor type for the dataset of type: " + + static_cast(type)); + }; } /** @@ -57,16 +57,16 @@ std::unique_ptr createDataBuffer(DatasetType type, */ std::unique_ptr createDataBuffer(DatasetType type, datagen_cb cb, void *user_data) { - if (type != DatasetType::GENERATOR) - throw std::invalid_argument("Cannot create dataset with generator " - "callbacks with the given dataset type"); - - std::unique_ptr dataset = createDataBuffer(type); - - if (dataset->setGeneratorFunc(cb, user_data) != ML_ERROR_NONE) - throw std::invalid_argument("Invalid train data generator"); - - return dataset; + switch (type) { + case DatasetType::GENERATOR: + return std::make_unique(cb, user_data); + case DatasetType::UNKNOWN: + [[fallthrough]]; + default: + throw std::invalid_argument( + "Unsupported constructor type for the dataset of type: " + + static_cast(type)); + }; } } // namespace nntrainer diff --git a/nntrainer/dataset/databuffer_file.h b/nntrainer/dataset/databuffer_file.h index e15f39c..44ff9ee 100644 --- a/nntrainer/dataset/databuffer_file.h +++ b/nntrainer/dataset/databuffer_file.h @@ -31,6 +31,7 @@ #include #include +#include namespace nntrainer { @@ -47,6 +48,14 @@ public: DataBufferFromDataFile() : DataBuffer(DatasetType::FILE) {} /** + * @brief Constructor + */ + DataBufferFromDataFile(const std::string &path) : DataBufferFromDataFile() { + NNTR_THROW_IF(setDataFile(path) != ML_ERROR_NONE, std::invalid_argument) + << "invalid train file, path: " << path; + } + + /** * @brief Destructor */ ~DataBufferFromDataFile() = default; diff --git a/nntrainer/dataset/databuffer_func.h b/nntrainer/dataset/databuffer_func.h index 845b174..74fea22 100644 --- a/nntrainer/dataset/databuffer_func.h +++ b/nntrainer/dataset/databuffer_func.h @@ -47,6 +47,15 @@ public: DataBufferFromCallback() : DataBuffer(DatasetType::GENERATOR) {} /** + * @brief Construct a new Data Buffer From Callback object + * + */ + DataBufferFromCallback(datagen_cb func, void *user_data = nullptr) : + DataBuffer(DatasetType::GENERATOR) { + setGeneratorFunc(func, user_data); + } + + /** * @brief Destructor */ ~DataBufferFromCallback() = default; diff --git a/nntrainer/models/model_loader.cpp b/nntrainer/models/model_loader.cpp index 6301344..eec6f2d 100644 --- a/nntrainer/models/model_loader.cpp +++ b/nntrainer/models/model_loader.cpp @@ -208,13 +208,6 @@ int ModelLoader::loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model) { return ML_ERROR_INVALID_PARAMETER; } - model.data_buffers[static_cast(DatasetDataUsageType::DATA_TRAIN)] = - nntrainer::createDataBuffer(DatasetType::FILE); - model.data_buffers[static_cast(DatasetDataUsageType::DATA_VAL)] = - nntrainer::createDataBuffer(DatasetType::FILE); - model.data_buffers[static_cast(DatasetDataUsageType::DATA_TEST)] = - nntrainer::createDataBuffer(DatasetType::FILE); - /// @todo ini bufferSize -> buffer_size to unify std::string bufsizepros("buffer_size="); bufsizepros += iniparser_getstring(ini, "DataSet:BufferSize", "1"); @@ -227,12 +220,16 @@ int ModelLoader::loadDatasetConfigIni(dictionary *ini, NeuralNetwork &model) { return required ? ML_ERROR_INVALID_PARAMETER : ML_ERROR_NONE; } - auto dbuffer = std::static_pointer_cast( - model.data_buffers[static_cast(dt)]); - - dbuffer->setProperty({bufsizepros}); + try { + model.data_buffers[static_cast(dt)] = + createDataBuffer(DatasetType::FILE, resolvePath(path).c_str()); + model.data_buffers[static_cast(dt)]->setProperty({bufsizepros}); + } catch (...) { + ml_loge("path is not valid, path: %s", resolvePath(path).c_str()); + return ML_ERROR_INVALID_PARAMETER; + } - return dbuffer->setDataFile(resolvePath(path)); + return ML_ERROR_NONE; }; status = diff --git a/test/ccapi/unittest_ccapi.cpp b/test/ccapi/unittest_ccapi.cpp index e4e1843..5bfa2f0 100644 --- a/test/ccapi/unittest_ccapi.cpp +++ b/test/ccapi/unittest_ccapi.cpp @@ -151,9 +151,9 @@ TEST(ccapi_dataset, construct_01_n) { /** * @brief Neural Network Dataset Contruct Test */ -TEST(ccapi_dataset, construct_02_p) { - EXPECT_NO_THROW(ml::train::createDataset(ml::train::DatasetType::GENERATOR)); - EXPECT_NO_THROW(ml::train::createDataset(ml::train::DatasetType::FILE)); +TEST(ccapi_dataset, construct_02_n) { + EXPECT_THROW(ml::train::createDataset(ml::train::DatasetType::GENERATOR), + std::invalid_argument); } static nntrainer::IniSection model_base("Model", "Type = NeuralNetwork" -- 2.7.4