From: Jihoon Lee Date: Tue, 20 Oct 2020 10:36:37 +0000 (+0900) Subject: [Test] Add unittest_nntrainer_models X-Git-Tag: accepted/tizen/unified/20201120.125454~61 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=99c6df6102532e78eb63750073454ed74f7bab62;p=platform%2Fcore%2Fml%2Fnntrainer.git [Test] Add unittest_nntrainer_models This patch generates `NodeWatcher` and `GraphWatcher` to make a test for unittest_nntrainer_model test. **Changes proposed in this PR:** - Add multi iteration test - Add gtest scafolding **Additional Patch will be followed** - Handle loss + activation merge scenario - Add param test - Add mnist as a test - Add final inference result **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Jihoon Lee --- diff --git a/test/unittest/meson.build b/test/unittest/meson.build index 728367d..88e8c3f 100644 --- a/test/unittest/meson.build +++ b/test/unittest/meson.build @@ -21,7 +21,8 @@ test_target = [ 'unittest_nntrainer_tensor', 'unittest_util_func', 'unittest_databuffer_file', - 'unittest_nntrainer_modelfile' + 'unittest_nntrainer_modelfile', + 'unittest_nntrainer_models' ] foreach target: test_target diff --git a/test/unittest/unittest_nntrainer_models.cpp b/test/unittest/unittest_nntrainer_models.cpp new file mode 100644 index 0000000..9cc2656 --- /dev/null +++ b/test/unittest/unittest_nntrainer_models.cpp @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright (C) 2020 Jihoon Lee + * + * @file unittest_nntrainer_models.cpp + * @date 19 Oct 2020 + * @brief Model multi iteration, itegrated test + * @see https://github.com/nnstreamer/nntrainer + * @author Jihoon Lee + * @bug No known bugs except for NYI items + * + */ +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +using NodeType = nntrainer::NeuralNetwork::NodeType; +using FlatGraphType = nntrainer::NeuralNetwork::FlatGraphType; + +/** + * @brief verify tensor to the reference and throw if not match to stop + * + * @param actual actual tensor + * @param expected reference tensor + * @param error_msg error msg to print if not match + */ +void verify(const nntrainer::Tensor &actual, const nntrainer::Tensor &expected, + const std::string &error_msg) { + if (actual != expected) { + std::cout + << "============================================================\n"; + std::cout << "current " << actual << "expected " << expected; + throw std::invalid_argument(error_msg.c_str()); + } +} + +/** + * @brief NodeWatcher has an operation Node. This class monitors data in and out + * happening in the node. + * + */ +class NodeWatcher { +public: + /** + * @brief Construct a new Node Watcher object + * + */ + NodeWatcher() {} + + /** + * @brief Construct a new Node Watcher object + * + * @param node node to watch. + */ + NodeWatcher(const NodeType &node) : node(node) { + unsigned int num_weights = node->getNumWeights(); + node->setTrainable(false); + + expected_input = nntrainer::Tensor(node->getInputDimension()); + + for (unsigned int i = 0; i < num_weights; ++i) { + const nntrainer::Weight &w = node->weightAt(i); + expected_weights.push_back(w); + } + + expected_output = nntrainer::Tensor(node->getOutputDimension()); + expected_dx = nntrainer::Tensor(node->getInputDimension()); + } + + /** + * @brief clones from expected weights to node->weights + * + */ + void cloneWeightsFromExpected() { + for (unsigned int i = 0; i < expected_weights.size(); ++i) { + node->weightAt(i) = expected_weights[i]; + } + } + + /** + * @brief forward node with verifying inputs/weights/outputs + * + * @param in input tensor + * @param iteration iteration + * @return nntrainer::sharedConstTensor + */ + nntrainer::sharedConstTensor forward(nntrainer::sharedConstTensor in, + int iteration); + + /** + * @brief forward loss node with verifying inputs/weights/outputs + * + * @param pred tensor predicted from the graph + * @param answer label tensor + * @param iteration iteration + * @return nntrainer::sharedConstTensor + */ + nntrainer::sharedConstTensor lossForward(nntrainer::sharedConstTensor pred, + nntrainer::sharedConstTensor answer, + int iteration); + + /** + * @brief backward pass of the node with verifying inputs/gradients/outputs + * + * @param deriv dervatives + * @param iteration iteration + * @param should_verify should verify the inputs/gradients/outputs + * @return nntrainer::sharedConstTensor + */ + nntrainer::sharedConstTensor backward(nntrainer::sharedConstTensor deriv, + int iteration, + bool should_verify = true); + + /** + * @brief verify weights of the current node + * + * @param error_msg error_msg to put when fail + */ + void verifyWeight(const std::string &error_msg); + + /** + * @brief verify gradient of the current node + * + * @param error_msg error_msg to put when fail + */ + void verifyGrad(const std::string &error_msg); + + /** + * @brief Get the Loss + * + * @return float loss + */ + float getLoss() { return node->getLoss(); } + + /** + * @brief read Node + * + * @param in input file stream + */ + void read(std::ifstream &in); + +private: + NodeType node; + nntrainer::Tensor expected_input; + nntrainer::Tensor expected_output; + nntrainer::Tensor expected_dx; + std::vector expected_weights; +}; + +class GraphWatcher { +public: + using WatchedFlatGraph = std::vector; + GraphWatcher(const std::string &config); + + void compareFor(const std::string &reference, + const nntrainer::TensorDim &label_shape, + unsigned int iterations); + +private: + void readIteration(std::ifstream &f); + + nntrainer::NeuralNetwork nn; + WatchedFlatGraph nodes; + NodeWatcher loss_node; + float expected_loss; +}; + +void NodeWatcher::read(std::ifstream &in) { + expected_input.read(in); + + /// @note below is harrasing the fact the tensor shares same base memory + /// it should better be getVariableRef() or somewhat equivalent + for (auto &i : expected_weights) { + i.getVariable().read(in); + } + + for (auto &i : expected_weights) { + i.getGradient().read(in); + } + + expected_output.read(in); + expected_dx.read(in); +} + +void NodeWatcher::verifyWeight(const std::string &error_msg) { + for (unsigned int i = 0; i < expected_weights.size(); ++i) { + verify(node->weightAt(i).getVariable(), expected_weights[i].getVariable(), + error_msg + " " + node->weightAt(i).getName() + "weight"); + } +} + +void NodeWatcher::verifyGrad(const std::string &error_msg) { + for (unsigned int i = 0; i < expected_weights.size(); ++i) { + verify(node->weightAt(i).getGradient(), expected_weights[i].getGradient(), + error_msg + " " + node->weightAt(i).getName() + "grad"); + } +} + +nntrainer::sharedConstTensor +NodeWatcher::forward(nntrainer::sharedConstTensor in, int iteration) { + std::stringstream ss; + ss << "forward failed at " << node->getName() << " at iteration " + << iteration; + std::string err_msg = ss.str(); + + verify(*in, expected_input, err_msg + " at input "); + nntrainer::sharedConstTensor out = node->forwarding(in); + verify(*out, expected_output, err_msg + " at output "); + return out; +} + +nntrainer::sharedConstTensor +NodeWatcher::lossForward(nntrainer::sharedConstTensor pred, + nntrainer::sharedConstTensor answer, int iteration) { + std::stringstream ss; + ss << "loss failed at " << node->getName() << " at iteration " << iteration; + std::string err_msg = ss.str(); + + nntrainer::sharedConstTensor out = + std::static_pointer_cast(node)->forwarding(pred, + answer); + + return out; +} + +nntrainer::sharedConstTensor +NodeWatcher::backward(nntrainer::sharedConstTensor deriv, int iteration, + bool should_verify) { + std::stringstream ss; + ss << "backward failed at " << node->getName() << " at iteration " + << iteration; + std::string err_msg = ss.str(); + + nntrainer::sharedConstTensor out = node->backwarding(deriv, iteration); + + if (should_verify) { + verify(*out, expected_dx, err_msg); + verifyGrad(err_msg); + } + + auto opt = node->getOptimizer(); + if (opt) { + opt->apply_gradients(node->getWeights(), node->getNumWeights(), iteration); + } + + return out; +} + +GraphWatcher::GraphWatcher(const std::string &config) { + nn = nntrainer::NeuralNetwork(); + nn.loadFromConfig(config); + nn.init(); + + FlatGraphType graph = nn.getFlatGraph(); + + for (auto it = graph.begin(); it != graph.end() - 1; ++it) { + nodes.push_back(NodeWatcher(*it)); + } + + loss_node = NodeWatcher(graph.back()); +} + +void GraphWatcher::compareFor(const std::string &reference, + const nntrainer::TensorDim &label_shape, + unsigned int iterations) { + std::ifstream ref(reference, std::ios_base::in | std::ios_base::binary); + + if (ref.bad()) { + throw std::runtime_error("ref is bad!"); + } + + nntrainer::Tensor in(nn.getInputDimension()); + nntrainer::Tensor lb(label_shape); + + in.read(ref); + lb.read(ref); + + auto prepareInitialWeight = [this]() { + std::for_each(nodes.begin(), nodes.end(), + [](NodeWatcher &n) { n.cloneWeightsFromExpected(); }); + }; + + auto matchWeightAfterUpdation = [this]() { + std::for_each(nodes.begin(), nodes.end(), [](NodeWatcher &n) { + n.verifyWeight("weight is diffrent after updation, check optimizer"); + }); + }; + + for (unsigned int iteration = 1; iteration <= iterations; ++iteration) { + nntrainer::sharedConstTensor input = MAKE_SHARED_TENSOR(in.clone()); + nntrainer::sharedConstTensor label = MAKE_SHARED_TENSOR(lb.clone()); + + readIteration(ref); + + iteration == 1 ? prepareInitialWeight() : matchWeightAfterUpdation(); + + /// forward pass + for (auto &i : nodes) + input = i.forward(input, iteration); + loss_node.lossForward(input, label, iteration); + + EXPECT_FLOAT_EQ(expected_loss, loss_node.getLoss()); + + /// backward pass and update weights + nntrainer::sharedConstTensor output = + loss_node.backward(label, iteration, false); + for (auto it = nodes.rbegin(); it != nodes.rend(); it++) + output = it->backward(output, iteration); + } + + /// note that last weight update is not checked up. this need to be fixed +} + +void GraphWatcher::readIteration(std::ifstream &f) { + for (auto &i : nodes) { + i.read(f); + } + + f.read((char *)&expected_loss, sizeof(float)); +} + +/** + * @brief Main gtest + */ +int main(int argc, char **argv) { + int result = -1; + + try { + testing::InitGoogleTest(&argc, argv); + } catch (...) { + std::cerr << "Error duing IniGoogleTest" << std::endl; + return 0; + } + + try { + result = RUN_ALL_TESTS(); + } catch (...) { + std::cerr << "Error duing RUN_ALL_TSETS()" << std::endl; + } + + return result; +}