From dc9fa87435cf3ceaac7d4ceb5a46f158f1d48d6d Mon Sep 17 00:00:00 2001 From: Jihoon Lee Date: Mon, 5 Apr 2021 16:11:03 +0900 Subject: [PATCH] [Ini] Implement ini deserializer This patch implements ini deserializer as well as changing the interface signature to get filepath. Many NYI parts will be implemented soon. **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Jihoon Lee --- jni/Android.mk | 1 + nntrainer/compiler/ini_interpreter.cpp | 148 +++++++++++++++++++++++++++++++++ nntrainer/compiler/ini_interpreter.h | 81 ++++++++++++++++++ nntrainer/compiler/interpreter.h | 11 +-- nntrainer/compiler/meson.build | 4 +- 5 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 nntrainer/compiler/ini_interpreter.cpp create mode 100644 nntrainer/compiler/ini_interpreter.h diff --git a/jni/Android.mk b/jni/Android.mk index a374c24..2856b6d 100644 --- a/jni/Android.mk +++ b/jni/Android.mk @@ -127,6 +127,7 @@ NNTRAINER_SRCS := $(NNTRAINER_ROOT)/nntrainer/models/neuralnet.cpp \ $(NNTRAINER_ROOT)/nntrainer/utils/util_func.cpp \ $(NNTRAINER_ROOT)/nntrainer/utils/parse_util.cpp \ $(NNTRAINER_ROOT)/nntrainer/utils/profiler.cpp \ + $(NNTRAINER_ROOT)/nntrainer/compiler/ini_interpreter.cpp \ $(NNTRAINER_ROOT)/nntrainer/app_context.cpp # Add tflite backbone building diff --git a/nntrainer/compiler/ini_interpreter.cpp b/nntrainer/compiler/ini_interpreter.cpp new file mode 100644 index 0000000..6ab80a8 --- /dev/null +++ b/nntrainer/compiler/ini_interpreter.cpp @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2021 Jihoon Lee + * + * @file ini_interpreter.cpp + * @date 02 April 2021 + * @brief NNTrainer Ini Interpreter (partly moved from model_loader.c) + * @see https://github.com/nnstreamer/nntrainer + * @author Jijoong Moon + * @author Parichay Kapoor + * @author Jihoon Lee + * @bug No known bugs except for NYI items + */ +#include + +#include + +#include +#include +#include +#include + +static constexpr const char *FUNC_TAG = "[IniInterpreter] "; + +static constexpr const char *UNKNOWN_STR = "UNKNOWN"; +static constexpr const char *NONE_STR = "NONE"; +static constexpr const char *MODEL_STR = "model"; +static constexpr const char *DATASET_STR = "dataset"; +static constexpr const char *OPTIMIZER_STR = "optimizer"; + +namespace nntrainer { + +namespace { + +/** @todo: + * 1. deprecate tag dispatching along with #1072 + * 2. deprecate getMergeableGraph (extendGraph should accept graph itself) + */ +class PlainLayer {}; /**< Plain Layer tag */ +class BackboneLayer {}; /**< Backbone Layer tag */ + +template +static std::shared_ptr section2layer(dictionary *ini, + const std::string &sec_name) { + /// NYI! + return nullptr; +} + +static std::shared_ptr +section2graph(dictionary *ini, const std::string &sec_name) { + /// NYI! + return nullptr; +} + +bool graphSupported(const std::string &backbone_name) { + /// NYI! + return true; +} + +std::vector> +getMergeableGraph(std::shared_ptr) { + return {}; +}; + +} // namespace + +void IniGraphInterpreter::serialize( + std::shared_ptr representation, + const std::string &out) {} + +std::shared_ptr +IniGraphInterpreter::deserialize(const std::string &in) { + + NNTR_THROW_IF(in.empty(), std::invalid_argument) + << FUNC_TAG << "given in file is empty"; + + NNTR_THROW_IF(!isFileExist(in), std::invalid_argument) + << FUNC_TAG << "given ini file does not exist"; + + dictionary *ini = iniparser_load(in.c_str()); + NNTR_THROW_IF(!ini, std::runtime_error) << "loading ini failed"; + + auto freedict = [ini] { iniparser_freedict(ini); }; + + /** Get number of sections in the file */ + int num_ini_sec = iniparser_getnsec(ini); + NNTR_THROW_IF_CLEANUP(num_ini_sec < 0, std::invalid_argument, freedict) + << FUNC_TAG << "invalid number of sections."; + + std::shared_ptr graph = + std::make_shared(); + + try { + for (int idx = 0; idx < num_ini_sec; ++idx) { + auto sec_name_ = iniparser_getsecname(ini, idx); + NNTR_THROW_IF_CLEANUP(!sec_name_, std::runtime_error, freedict) + << "parsing a section name returned error, filename: " << in + << "idx: " << idx; + + ml_logd("probing section_name: %s", sec_name_); + std::string sec_name(sec_name_); + + if (istrequal(sec_name, MODEL_STR) || istrequal(sec_name, DATASET_STR) || + istrequal(sec_name, OPTIMIZER_STR)) { + /// dedicated sections so skip + continue; + } + /** Parse all the layers defined as sections in order */ + std::shared_ptr layer; + + /** + * If this section is a backbone, load backbone section from this + * @note The order of backbones in the ini file defines the order on the + * backbones in the model graph + */ + const char *backbone_path = + iniparser_getstring(ini, (sec_name + ":Backbone").c_str(), UNKNOWN_STR); + + const std::string &backbone = pathResolver(backbone_path); + if (graphSupported(backbone)) { + auto g = section2graph(ini, sec_name); + + /// @todo: deprecate this. We should extend graph from a graph + auto tmp = getMergeableGraph(g); + graph->extendGraph(tmp, sec_name); + continue; + } + + if (backbone_path == UNKNOWN_STR) { + layer = section2layer(ini, sec_name); + } else { + /// @todo deprecate this as well with #1072 + layer = section2layer(ini, sec_name); + } + + graph->addLayer(layer); + } + } catch (...) { + /** clean up and rethrow */ + freedict(); + throw; + } + + freedict(); + return graph; +} + +} // namespace nntrainer diff --git a/nntrainer/compiler/ini_interpreter.h b/nntrainer/compiler/ini_interpreter.h new file mode 100644 index 0000000..e5b8c59 --- /dev/null +++ b/nntrainer/compiler/ini_interpreter.h @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2021 Jihoon Lee + * + * @file ini_interpreter.h + * @date 02 April 2021 + * @brief NNTrainer Ini Interpreter + * @see https://github.com/nnstreamer/nntrainer + * @author Jihoon Lee + * @bug No known bugs except for NYI items + */ +#include +#include + +#include + +#include +#include + +#ifndef __INI_INTERPRETER_H__ +#define __INI_INTERPRETER_H__ + +namespace nntrainer { + +/** + * @brief ini graph interpreter class + * + */ +class IniGraphInterpreter : public GraphInterpreter { +public: + /** + * @brief Construct a new Ini Graph Interpreter object + * + * @param pathResolver_ path resolver function to be used + */ + IniGraphInterpreter( + std::function pathResolver_ = + [](const std::string &path) { return path; }) : + pathResolver(pathResolver_) {} + + virtual ~IniGraphInterpreter(){}; + + /** + * @copydoc GraphInterpreter::serialize(const std::string &out) + */ + void serialize(std::shared_ptr representation, + const std::string &out) override; + + /** + * @copydoc GraphInterpreter::deserialize(const std::string &in) + */ + std::shared_ptr + deserialize(const std::string &in) override; + +private: + /** + * @brief Create a Layer From Section object + * + * @param ini ini if throw, ini will be freed. + * @param section section name + * @return std::shared_ptr layer + */ + std::shared_ptr loadLayerConfig(dictionary *ini, + const std::string §ion); + + /** + * @brief Create a Layer From Backbone Config + * + * @param ini ini if throw, ini will be freed. + * @param section section name + * @return std::shared_ptr layer + */ + std::shared_ptr loadBackboneConfigIni(dictionary *ini, + const std::string §ion); + + std::function pathResolver; +}; + +} // namespace nntrainer + +#endif // __INI_INTERPRETER_H__ diff --git a/nntrainer/compiler/interpreter.h b/nntrainer/compiler/interpreter.h index 3a7cc87..23487ca 100644 --- a/nntrainer/compiler/interpreter.h +++ b/nntrainer/compiler/interpreter.h @@ -9,6 +9,7 @@ * @see https://github.com/nnstreamer/nntrainer * @author Jihoon Lee * @bug No known bugs except for NYI items + * @note The boundary of graph interpreter is restricted to graph only. * @details * Graph is convertible either from a file, representation by a appropriate interpreter @@ -38,8 +39,8 @@ #ifndef __INTERPRETER_H__ #define __INTERPRETER_H__ -#include #include +#include #include @@ -58,20 +59,20 @@ public: * @brief serialize graph to a stream * * @param representation graph representation - * @param out outstream to serialize graph + * @param out output file name */ virtual void serialize(std::shared_ptr representation, - std::ostream &out) = 0; + const std::string &out) = 0; /** * @brief deserialize graph from a stream * - * @param in in stream to deserialize + * @param in input file name * @return GraphRepresentation graph representation */ virtual std::shared_ptr - deserialize(std::istream &in) = 0; + deserialize(const std::string &in) = 0; }; } // namespace nntrainer diff --git a/nntrainer/compiler/meson.build b/nntrainer/compiler/meson.build index f95d055..0f484c2 100644 --- a/nntrainer/compiler/meson.build +++ b/nntrainer/compiler/meson.build @@ -1,4 +1,6 @@ -compiler_sources = [] +compiler_sources = [ + 'ini_interpreter.cpp' +] compiler_headers = [] -- 2.7.4