From d4dc2f27299584e6cd71ff3c7cf801e8c132a781 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Fri, 23 Oct 2020 12:06:04 +0900 Subject: [PATCH] [backbone] Add unittests for backbone Added unittests for backbone where model constructed with and without backbone are matched to be equivalent. Corresponding bug fixes are also added. See also #660 **Self evaluation:** 1. Build test: [x]Passed [ ]Failed [ ]Skipped 2. Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Parichay Kapoor --- nntrainer/include/neuralnet.h | 1 + nntrainer/src/model_loader.cpp | 4 +- nntrainer/src/neuralnet.cpp | 2 +- test/include/nntrainer_test_util.h | 33 +++++- test/unittest/unittest_nntrainer_modelfile.cpp | 147 ++++++++++++++++++++++++- 5 files changed, 176 insertions(+), 11 deletions(-) diff --git a/nntrainer/include/neuralnet.h b/nntrainer/include/neuralnet.h index 3d9525d..56cb6cc 100644 --- a/nntrainer/include/neuralnet.h +++ b/nntrainer/include/neuralnet.h @@ -245,6 +245,7 @@ public: /** * @brief join passed graph into the existing graph model * @param[in] graph graph to be added/to extend + * @param[in] prefix prefix added to names of layers from this graph * @note It is assumed that this model is valid by itself * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. diff --git a/nntrainer/src/model_loader.cpp b/nntrainer/src/model_loader.cpp index 72848f0..4d97d60 100644 --- a/nntrainer/src/model_loader.cpp +++ b/nntrainer/src/model_loader.cpp @@ -305,8 +305,8 @@ int ModelLoader::loadFromIni(std::string ini_file, NeuralNetwork &model, * backbones in the model graph */ const char *backbone = - iniparser_getstring(ini, (sec_name + ":Backbone").c_str(), nullptr); - if (backbone != nullptr) { + iniparser_getstring(ini, (sec_name + ":Backbone").c_str(), unknown); + if (backbone != unknown) { loadBackboneConfigIni(backbone, model, sec_name); continue; } diff --git a/nntrainer/src/neuralnet.cpp b/nntrainer/src/neuralnet.cpp index 4dd4520..7585691 100644 --- a/nntrainer/src/neuralnet.cpp +++ b/nntrainer/src/neuralnet.cpp @@ -632,7 +632,7 @@ void NeuralNetwork::ensureName(NodeType layer, const std::string &prefix, /** If just prefix with layer name makes it unique - directly set the name */ if (!orig_name_empty) { std::string direct_name = prefix + orig_name; - if (layer_names.find(direct_name) != layer_names.end()) { + if (layer_names.find(direct_name) == layer_names.end()) { layer->setName(direct_name); return; } diff --git a/test/include/nntrainer_test_util.h b/test/include/nntrainer_test_util.h index 770894f..6303c26 100644 --- a/test/include/nntrainer_test_util.h +++ b/test/include/nntrainer_test_util.h @@ -25,14 +25,15 @@ #define __NNTRAINER_TEST_UTIL_H__ #ifdef __cplusplus -#include "nntrainer_log.h" #include #include +#include + #include #include +#include #include #include -#include #define tolerance 10e-5 @@ -93,6 +94,10 @@ public: IniSection operator+(const std::string &s) { return IniSection(*this) += s; } + void rename(const std::string &name) { section_name = name; } + + std::string getName() { return section_name; } + private: void setEntry(const std::unordered_map &_entry) { for (auto &it : _entry) { @@ -151,7 +156,17 @@ public: * @return std::ofstream ini file stream */ std::ofstream getIni() { - std::ofstream out(getIniName().c_str()); + return getIni(getIniName().c_str(), std::ios_base::out); + } + + /** + * @brief Get the Ini object with given mode + * + * @return std::ofstream ini file stream + */ + static std::ofstream getIni(const char *filename, + std::ios_base::openmode mode) { + std::ofstream out(filename, mode); if (!out.good()) { throw std::runtime_error("cannot open ini"); } @@ -160,10 +175,11 @@ public: /** * @brief save ini to a file - * */ - void save_ini() { - std::ofstream out = getIni(); + static void save_ini(const char *filename, std::vector sections, + std::ios_base::openmode mode = std::ios_base::out) { + std::ofstream out = IniTestWrapper::getIni(filename, mode); + for (auto &it : sections) { it.print(std::cout); std::cout << std::endl; @@ -175,6 +191,11 @@ public: } /** + * @brief save ini to a file + */ + void save_ini() { save_ini(getIniName().c_str(), sections); } + + /** * @brief erase ini * */ diff --git a/test/unittest/unittest_nntrainer_modelfile.cpp b/test/unittest/unittest_nntrainer_modelfile.cpp index 21b58cf..47baa49 100644 --- a/test/unittest/unittest_nntrainer_modelfile.cpp +++ b/test/unittest/unittest_nntrainer_modelfile.cpp @@ -11,9 +11,11 @@ */ #include + #include +#include -#include "nntrainer_test_util.h" +#include namespace initest { typedef enum { @@ -26,6 +28,12 @@ class nntrainerIniTest : public ::testing::TestWithParam< std::tuple> { +public: + static void save_ini(const char *filename, std::vector sections, + std::ios_base::openmode mode = std::ios_base::out) { + IniTestWrapper::save_ini(filename, sections, mode); + } + protected: virtual void SetUp() { name = std::string(std::get<0>(GetParam())); @@ -176,6 +184,13 @@ static IniSection conv2d("conv2d", "Type = conv2d |" "stride = 1,1 |" "padding = 0,0 |"); +static IniSection input2d("inputlayer", "Type = input |" + "Input_Shape = 3:100:100"); + +static IniSection backbone_random("block1", "backbone = random.ini"); + +static IniSection backbone_valid("block1", "backbone = base.ini"); + static int SUCCESS = 0; static int LOADFAIL = initest::LOAD; static int INITFAIL = initest::INIT; @@ -246,7 +261,9 @@ INSTANTIATE_TEST_CASE_P( /**< negative: dataset is not complete (5 negative cases) */ mkIniTc("no_trainingSet_n", {nw_adam, dataset + "-TrainData", input, out}, ALLFAIL), - mkIniTc("no_labelSet_n", {nw_adam, dataset + "-LabelData", input, out}, ALLFAIL) + mkIniTc("no_labelSet_n", {nw_adam, dataset + "-LabelData", input, out}, ALLFAIL), + + mkIniTc("backbone_filemissing_n", {nw_adam, dataset + "-LabelData", input, out}, ALLFAIL) /// #if gtest_version <= 1.7.0 )); /// #else gtest_version > 1.8.0 @@ -257,6 +274,132 @@ INSTANTIATE_TEST_CASE_P( // clang-format on /** + * @brief Ini file unittest with backbone with wrong file + */ +TEST(nntrainerIniTest, backbone_n_01) { + const char *ini_name = "backbone_n1.ini"; + nntrainerIniTest::save_ini(ini_name, {nw_base, backbone_random}); + nntrainer::NeuralNetwork NN; + + EXPECT_EQ(NN.loadFromConfig(ini_name), ML_ERROR_INVALID_PARAMETER); +} + +/** + * @brief Ini file unittest with backbone with empty backbone + */ +TEST(nntrainerIniTest, backbone_n_02) { + const char *ini_name = "backbone_n2.ini"; + nntrainerIniTest::save_ini("base.ini", {nw_base}); + nntrainerIniTest::save_ini(ini_name, {nw_base, backbone_valid}); + nntrainer::NeuralNetwork NN; + + EXPECT_EQ(NN.loadFromConfig(ini_name), ML_ERROR_INVALID_PARAMETER); +} + +/** + * @brief Ini file unittest with backbone with normal backbone + */ +TEST(nntrainerIniTest, backbone_p_03) { + const char *ini_name = "backbone_p3.ini"; + nntrainerIniTest::save_ini("base.ini", {nw_base, batch_normal}); + nntrainerIniTest::save_ini(ini_name, {nw_base, backbone_valid}); + nntrainer::NeuralNetwork NN; + + EXPECT_EQ(NN.loadFromConfig(ini_name), ML_ERROR_NONE); +} + +/** + * @brief Ini file unittest with backbone without model parameters + */ +TEST(nntrainerIniTest, backbone_p_04) { + const char *ini_name = "backbone_p4.ini"; + nntrainerIniTest::save_ini("base.ini", {flatten, conv2d}); + nntrainerIniTest::save_ini(ini_name, {nw_base, backbone_valid}); + nntrainer::NeuralNetwork NN; + + EXPECT_EQ(NN.loadFromConfig(ini_name), ML_ERROR_NONE); +} + +/** + * @brief Ini file unittest matching model with and without backbone + */ +TEST(nntrainerIniTest, backbone_p_05) { + const char *bb_use_ini_name = "backbone_made.ini"; + const char *direct_ini_name = "direct_made.ini"; + + /** Create a backbone.ini */ + nntrainerIniTest::save_ini("base.ini", {nw_adam, conv2d}); + + /** Create a model of 4 conv layers using backbone */ + std::string backbone_valid_orig_name = backbone_valid.getName(); + + nntrainerIniTest::save_ini(bb_use_ini_name, + {nw_sgd, input2d, backbone_valid}); + backbone_valid.rename("block2"); + nntrainerIniTest::save_ini(bb_use_ini_name, {backbone_valid}, + std::ios_base::app); + backbone_valid.rename("block3"); + nntrainerIniTest::save_ini(bb_use_ini_name, {backbone_valid}, + std::ios_base::app); + backbone_valid.rename("block4"); + nntrainerIniTest::save_ini(bb_use_ini_name, {backbone_valid}, + std::ios_base::app); + + backbone_valid.rename(backbone_valid_orig_name); + + nntrainer::NeuralNetwork NN_backbone; + EXPECT_EQ(NN_backbone.loadFromConfig(bb_use_ini_name), ML_ERROR_NONE); + EXPECT_EQ(NN_backbone.init(), ML_ERROR_NONE); + + /** + * Model defined in backbone with adam with lr 0.0001 does not affect the + * final model to be made using the backbone. + */ + EXPECT_EQ(NN_backbone.getLearningRate(), 1); + + /** Create the same model directly without using backbone */ + std::string conv2d_orig_name = conv2d.getName(); + + nntrainerIniTest::save_ini(direct_ini_name, {nw_sgd, input2d}); + conv2d.rename("block1conv2d"); + nntrainerIniTest::save_ini(direct_ini_name, {conv2d}, std::ios_base::app); + conv2d.rename("block2conv2d"); + nntrainerIniTest::save_ini(direct_ini_name, {conv2d}, std::ios_base::app); + conv2d.rename("block3conv2d"); + nntrainerIniTest::save_ini(direct_ini_name, {conv2d}, std::ios_base::app); + conv2d.rename("block4conv2d"); + nntrainerIniTest::save_ini(direct_ini_name, {conv2d}, std::ios_base::app); + + conv2d.rename(conv2d_orig_name); + + nntrainer::NeuralNetwork NN_direct; + EXPECT_EQ(NN_direct.loadFromConfig(direct_ini_name), ML_ERROR_NONE); + EXPECT_EQ(NN_direct.init(), ML_ERROR_NONE); + + /** Summary of both the models must match precisely */ + NN_backbone.printPreset(std::cout, ML_TRAIN_SUMMARY_MODEL); + NN_direct.printPreset(std::cout, ML_TRAIN_SUMMARY_MODEL); + + EXPECT_EQ(NN_backbone.getInputDimension(), NN_direct.getInputDimension()); + EXPECT_EQ(NN_backbone.getOutputDimension(), NN_direct.getOutputDimension()); + + auto flat_backbone = NN_backbone.getFlatGraph(); + auto flat_direct = NN_direct.getFlatGraph(); + EXPECT_EQ(flat_backbone.size(), flat_direct.size()); + + for (size_t idx = 0; idx < flat_backbone.size(); idx++) { + EXPECT_EQ(flat_backbone[idx]->getType(), flat_direct[idx]->getType()); + EXPECT_EQ(flat_backbone[idx]->getInputDimension(), + flat_direct[idx]->getInputDimension()); + EXPECT_EQ(flat_backbone[idx]->getOutputDimension(), + flat_direct[idx]->getOutputDimension()); + EXPECT_EQ(flat_backbone[idx]->getActivationType(), + flat_direct[idx]->getActivationType()); + EXPECT_EQ(flat_backbone[idx]->getName(), flat_direct[idx]->getName()); + } +} + +/** * @brief Main gtest */ int main(int argc, char **argv) { -- 2.7.4