From 1b34516a949f6123071999e86b7cbbc69b4d918c Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Tue, 16 Mar 2021 12:28:09 +0900 Subject: [PATCH] [graph] Remove layers from graph Update graph to remove layers and have a single data structure for maintainence purposes. Now, adj represents the nodes list which is unsorted. Sorted list of nodes is available after topological sort in Sorted. Additional Bug fixes: - getSortedLayerNode was giving out wrong indices, its fixed now - topological sort now empties sorted list before adding new elements - other minor fixes to make adj replace layers **Self evaluation:** 1. Build test: [x]Passed [ ]Failed [ ]Skipped 2. Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Parichay Kapoor --- nntrainer/graph/network_graph.cpp | 172 +++++++++++--------- nntrainer/graph/network_graph.h | 30 ++-- nntrainer/models/neuralnet.cpp | 7 +- test/unittest/unittest_nntrainer_models.cpp | 1 + 4 files changed, 120 insertions(+), 90 deletions(-) diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp index 400a73d2..e9b022de 100644 --- a/nntrainer/graph/network_graph.cpp +++ b/nntrainer/graph/network_graph.cpp @@ -39,10 +39,11 @@ static const std::vector in_place_layers = { void NetworkGraph::updateNameInLayers(const std::string &cname, const std::string &name) { - for (unsigned int i = 0; i < layers.size(); ++i) { - for (unsigned int j = 0; j < layers[i]->input_layers.size(); ++j) { - if (istrequal(layers[i]->input_layers[j], cname)) { - layers[i]->input_layers[j] = name; + for (unsigned int i = 0; i < adj.size(); ++i) { + auto &layer = adj[i].front().layer; + for (unsigned int j = 0; j < layer->input_layers.size(); ++j) { + if (istrequal(layer->input_layers[j], cname)) { + layer->input_layers[j] = name; return; } } @@ -96,14 +97,7 @@ LayerNode &NetworkGraph::getLayerNode(unsigned int ith) { } LayerNode &NetworkGraph::getSortedLayerNode(unsigned int ith) { - - for (unsigned int i = 0; i < Sorted.size(); ++i) { - if (Sorted[i].index == ith) { - return Sorted[i]; - } - } - - throw std::invalid_argument("Cannot find Layer"); + return Sorted[ith]; } void NetworkGraph::countNonTrainableLayersAtBegin() { @@ -119,6 +113,7 @@ void NetworkGraph::countNonTrainableLayersAtBegin() { void NetworkGraph::topologicalSort() { std::stack Stack; std::vector visited(num_node); + Sorted.clear(); std::fill(visited.begin(), visited.end(), false); @@ -188,6 +183,8 @@ int NetworkGraph::realizeMultiInputType(Layer ¤t) { for (unsigned int i = 0; i < current.input_layers.size(); ++i) layer->input_layers.push_back(current.input_layers[i]); + // TODO: update previous layer names + current.setNumInputs(1); current.input_layers.clear(); current.input_layers.push_back(layer->getName()); @@ -213,6 +210,8 @@ int NetworkGraph::realizeFlattenType(Layer ¤t) { ensureName(layer, current.getName()); + // TODO: update previous layer names + layer->setNumInputs(1); layer->input_layers.clear(); layer->input_layers.push_back(current.getName()); @@ -268,33 +267,44 @@ int NetworkGraph::realizeActivationType(Layer ¤t) { current.output_layers.clear(); current.output_layers.push_back(layer->getName()); - addLayerNode(layer); - updateNameInLayers(current.getName(), layer->getName()); + addLayerNode(layer); return status; } int NetworkGraph::addLossLayer(const LossType loss_type) { + int status = ML_ERROR_NONE; + + if (Sorted.back().layer->getType() == LossLayer::type) + return status; + + if (loss_type == LossType::LOSS_NONE) { + status = ML_ERROR_INVALID_PARAMETER; + NN_RETURN_STATUS(); + } + LossType updated_loss_type = loss_type; if (num_node == 0) { status = ML_ERROR_INVALID_PARAMETER; NN_RETURN_STATUS(); } + LayerNode last_node = Sorted.back(); if (updated_loss_type == LossType::LOSS_ENTROPY) { - if (getLayerNode(num_node - 1).layer->getType() != "activation") { + if (last_node.layer->getType() != "activation") { ml_loge("Error: Cross Entropy need last layer to have softmax or sigmoid " "activation."); return ML_ERROR_NOT_SUPPORTED; } - LayerNode act_layer_node = getLayerNode(num_node - 1); + LayerNode last_node = Sorted.back(); adj.pop_back(); + Sorted.pop_back(); num_node--; - switch (act_layer_node.layer->getActivationType()) { + switch (last_node.layer->getActivationType()) { case ActivationType::ACT_SIGMOID: updated_loss_type = LossType::LOSS_ENTROPY_SIGMOID; break; @@ -307,13 +317,12 @@ int NetworkGraph::addLossLayer(const LossType loss_type) { } } - std::string input_str = getLayerNode(num_node - 1).layer->getName(); + last_node = Sorted.back(); + std::string input_str = last_node.layer->getName(); std::shared_ptr layer = std::make_unique(); - ensureName(layer); - LayerNode last_node = getLayerNode(num_node - 1); last_node.layer->setNumOutputs(1); last_node.layer->output_layers.clear(); last_node.layer->output_layers.push_back(layer->getName()); @@ -331,44 +340,48 @@ int NetworkGraph::addLossLayer(const LossType loss_type) { NN_RETURN_STATUS(); addLayerNode(layer); + setEdge(adj.size() - 1); + topologicalSort(); return ML_ERROR_NONE; } void NetworkGraph::setOutputLayers() { - if (layers.back()->getNumOutputs() > 0 && - layers.back()->output_layers.size() > 0) { + auto &last_layer = adj.back().front().layer; + if (last_layer->getNumOutputs() > 0 && last_layer->output_layers.size() > 0) { throw std::runtime_error("last layer already has a output layer"); } - layers.back()->setNumOutputs(1); - layers.back()->output_layers.clear(); - layers.back()->output_layers.push_back("__exit__"); + last_layer->setNumOutputs(1); + last_layer->output_layers.clear(); + last_layer->output_layers.push_back("__exit__"); - for (unsigned int idx = 0; idx < layers.size(); ++idx) { - for (unsigned int i = 0; i < layers.size(); ++i) { - if (istrequal(layers[i]->getName(), layers[idx]->getName())) + for (unsigned int idx = 0; idx < adj.size(); ++idx) { + auto &layer_idx = adj[idx].front().layer; + for (unsigned int i = 0; i < adj.size(); ++i) { + auto &layer_i = adj[i].front().layer; + if (istrequal(layer_i->getName(), layer_idx->getName())) continue; - for (unsigned int j = 0; j < layers[i]->input_layers.size(); ++j) { - if (istrequal(layers[i]->input_layers[j], layers[idx]->getName())) { - for (unsigned int k = 0; k < layers[idx]->output_layers.size(); ++k) { - if (!istrequal(layers[idx]->output_layers[k], layers[i]->getName())) + for (unsigned int j = 0; j < layer_i->input_layers.size(); ++j) { + if (istrequal(layer_i->input_layers[j], layer_idx->getName())) { + for (unsigned int k = 0; k < layer_idx->output_layers.size(); ++k) { + if (!istrequal(layer_idx->output_layers[k], layer_i->getName())) continue; } - layers[idx]->output_layers.push_back(layers[i]->getName()); + layer_idx->output_layers.push_back(layer_i->getName()); } } } - if (layers[idx]->getNumOutputs() != layers[idx]->output_layers.size()) { - layers[idx]->setNumOutputs(layers[idx]->output_layers.size()); + if (layer_idx->getNumOutputs() != layer_idx->output_layers.size()) { + layer_idx->setNumOutputs(layer_idx->output_layers.size()); } } - for (auto idx = layers.begin(); idx < layers.end(); ++idx) { - if ((*idx)->output_layers.size() == 0) - throw std::invalid_argument("There is un-connected node"); + for (auto iter = adj.begin(); iter < adj.end(); ++iter) { + if ((*iter).front().layer->output_layers.size() == 0) + throw std::runtime_error("There is un-connected node"); } } @@ -402,12 +415,12 @@ int NetworkGraph::realizeMultiOutputType(Layer ¤t) { } int NetworkGraph::isCompilable() { - if (layers.empty()) { + if (adj.empty()) { ml_loge("Layer is empty"); return ML_ERROR_INVALID_PARAMETER; } - Layer &l = *layers[0]; + Layer &l = *adj[0].front().layer; /** Dimension of first layer must be known */ // TODO: move this to the input layer and not first layer @@ -429,14 +442,17 @@ int NetworkGraph::isCompilable() { return ML_ERROR_NONE; } -int NetworkGraph::setGraphNode(const LossType loss_type) { +int NetworkGraph::setGraphNode() { int status = ML_ERROR_NONE; setOutputLayers(); - for (unsigned int i = 0; i < layers.size(); ++i) { - Layer &l = *layers[i]; + size_t adj_size_before_realize = adj.size(); + /** This loop modifes adj. Get the size of adj preemptively. */ + + for (unsigned int i = 0; i < adj_size_before_realize; ++i) { + Layer &l = *adj[i].front().layer; ml_logd("layer name: %s", l.getName().c_str()); if (l.input_layers.size() < 1) { @@ -456,8 +472,6 @@ int NetworkGraph::setGraphNode(const LossType loss_type) { NN_RETURN_STATUS(); } - addLayerNode(layers[i]); - if (l.getType() != ActivationLayer::type) { status = realizeActivationType(l); NN_RETURN_STATUS(); @@ -474,12 +488,6 @@ int NetworkGraph::setGraphNode(const LossType loss_type) { } } - if (layers.back()->getType() != LossLayer::type && - loss_type != LossType::LOSS_NONE) { - status = addLossLayer(loss_type); - NN_RETURN_STATUS(); - } - return status; } @@ -496,26 +504,28 @@ LayerNode &NetworkGraph::getLayerNode(const std::string &layer_name) { throw std::invalid_argument("Cannot find Layer"); } -int NetworkGraph::setEdge() { - int status = ML_ERROR_NONE; +void NetworkGraph::setEdge(unsigned int adj_idx) { + std::list::iterator iter = adj[adj_idx].begin(); - std::list::iterator iter; - for (unsigned int i = 0; i < adj.size(); ++i) { - iter = adj[i].begin(); + /** TODO: remove this check - check if already initialized */ + if ((*iter).layer->getInputDimension()[0].getDataLen() != 0) + return; - if ((*iter).layer->getInputDimension()[0].getDataLen() != 0) + for (unsigned int j = 0; j < (*iter).layer->input_layers.size(); ++j) { + if (istrequal((*iter).layer->input_layers[j], "__data__")) continue; + unsigned int to_node_id = + getLayerNode((*iter).layer->input_layers[j]).index; + addEdge(to_node_id, (*iter)); + } +} - for (unsigned int j = 0; j < (*iter).layer->input_layers.size(); ++j) { - if (istrequal((*iter).layer->input_layers[j], "__data__")) - continue; - unsigned int to_node_id = - getLayerNode((*iter).layer->input_layers[j]).index; - addEdge(to_node_id, (*iter)); - } +int NetworkGraph::setEdge() { + for (unsigned int i = 0; i < adj.size(); ++i) { + setEdge(i); } - return status; + return ML_ERROR_NONE; } void NetworkGraph::setBatchSize(unsigned int batch_size) { @@ -558,23 +568,23 @@ NetworkGraph::getGraph(const std::string &input_layer, /** count layers after output layer */ unsigned int num_layers_remove_end = 0; if (!output_layer.empty()) { - for (auto iter = layers.rbegin(); iter != layers.rend(); iter++) { - if ((*iter)->getName() != output_layer) + for (auto iter = adj.rbegin(); iter != adj.rend(); iter++) { + if ((*iter).front().layer->getName() != output_layer) num_layers_remove_end++; else break; } } - if (num_layers_remove_end == layers.size()) + if (num_layers_remove_end == adj.size()) return {}; /** count layers before input layer */ unsigned int num_layers_remove_start = 0; if (!input_layer.empty()) { - for (auto iter = layers.begin(); - iter != layers.end() - num_layers_remove_end; iter++) { - if ((*iter)->getName() != input_layer) + for (auto iter = adj.begin(); iter != adj.end() - num_layers_remove_end; + iter++) { + if ((*iter).front().layer->getName() != input_layer) num_layers_remove_start++; else break; @@ -583,8 +593,17 @@ NetworkGraph::getGraph(const std::string &input_layer, /** copy the graph and return */ std::vector> ret; - std::copy(layers.begin() + num_layers_remove_start, - layers.end() - num_layers_remove_end, std::back_inserter(ret)); + std::transform(adj.begin() + num_layers_remove_start, + adj.end() - num_layers_remove_end, std::back_inserter(ret), + [](auto const &elem) { return elem.front().layer; }); + + return ret; +} + +std::vector> NetworkGraph::getLayers() { + std::vector> ret; + std::transform(adj.begin(), adj.end(), std::back_inserter(ret), + [](auto const &elem) { return elem.front().layer; }); return ret; } @@ -628,11 +647,12 @@ void NetworkGraph::extendGraph(std::vector> graph, } } - layers.push_back(layer); + addLayerNode(layer); } /** This allows connecting a layer to the backbone */ - sub_in_out.insert(std::make_pair(prefix, layers.back()->getName())); + sub_in_out.insert( + std::make_pair(prefix, adj.back().front().layer->getName())); } void NetworkGraph::addLayer(std::shared_ptr layer) { @@ -640,7 +660,7 @@ void NetworkGraph::addLayer(std::shared_ptr layer) { ensureName(layer); /** Insert the layer to the graph */ - layers.push_back(layer); + addLayerNode(layer); } void NetworkGraph::inPlaceOptimize(const std::string &layer_type, diff --git a/nntrainer/graph/network_graph.h b/nntrainer/graph/network_graph.h index 60a9783f..f4ba9fb9 100644 --- a/nntrainer/graph/network_graph.h +++ b/nntrainer/graph/network_graph.h @@ -81,6 +81,8 @@ public: * @note graph contains pointer to the actual nodes, which is not deeply * copied. * @retval current flat graph + * + * TODO: rename to getUnsortedLayers */ std::vector> getGraph(const std::string &input_layer, const std::string &output_layer); @@ -95,7 +97,7 @@ public: * @brief get if the graph is empty * @param[out] true if empty, else false */ - bool empty() { return layers.empty(); } + bool empty() { return adj.empty(); } /** * @brief Swap function for the class @@ -104,7 +106,6 @@ public: using std::swap; swap(lhs.num_node, rhs.num_node); - swap(lhs.layers, rhs.layers); swap(lhs.adj, rhs.adj); swap(lhs.Sorted, rhs.Sorted); swap(lhs.layer_names, rhs.layer_names); @@ -117,7 +118,6 @@ public: * @brief reset the graph */ void reset() { - layers.clear(); adj.clear(); Sorted.clear(); layer_names.clear(); @@ -153,9 +153,9 @@ public: * @retval Layer */ std::shared_ptr getLayer(const std::string &layer_name) { - for (auto iter = layers.begin(); iter != layers.end(); ++iter) { - if ((*iter)->getName() == layer_name) { - return *iter; + for (auto iter = adj.begin(); iter != adj.end(); ++iter) { + if ((*iter).front().layer->getName() == layer_name) { + return (*iter).front().layer; } } @@ -167,7 +167,7 @@ public: * @param[in] layer name * @retval Layer */ - std::vector> &getLayers() { return layers; } + std::vector> getLayers(); /** * @brief join passed graph into the existing graph model @@ -197,7 +197,7 @@ public: * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - int setGraphNode(const LossType loss_type); + int setGraphNode(); /** * @brief check and add Multi Input Layer : addition or concat Layer @@ -246,6 +246,13 @@ public: */ int setEdge(); + /** + * @brief make connection for the given node idx + * @retval #ML_ERROR_NONE Successful. + * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + */ + void setEdge(unsigned int adj_idx); + /** * @brief set batch size * @param[in] batch size @@ -306,8 +313,9 @@ public: */ NetworkGraph ©(NetworkGraph &from) { if (this != &from) { - for (unsigned int i = 0; i < layers.size(); i++) - layers[i]->copy(from.layers[i]); + // TODO: this assumes elements already in layers/adj, solve that + for (unsigned int i = 0; i < adj.size(); i++) + adj[i].front().layer->copy(from.adj[i].front().layer); } return *this; } @@ -333,8 +341,6 @@ private: std::map sub_in_out; /** This is map to identify input and output layer name of subgraph */ - std::vector> - layers; /**< vector for store layer pointers */ unsigned int num_node; /**< Total Number of Graph Nodes */ std::vector> adj; /**< Graph Structure */ std::vector Sorted; /**< Ordered Graph Node List */ diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp index e57fd555..63b3522a 100644 --- a/nntrainer/models/neuralnet.cpp +++ b/nntrainer/models/neuralnet.cpp @@ -148,7 +148,7 @@ int NeuralNetwork::compile() { ml_logd("Compile model"); - status = model_graph.setGraphNode(loss_type); + status = model_graph.setGraphNode(); NN_RETURN_STATUS(); status = model_graph.setEdge(); @@ -156,6 +156,9 @@ int NeuralNetwork::compile() { model_graph.topologicalSort(); + status = model_graph.addLossLayer(loss_type); + NN_RETURN_STATUS(); + auto &sorted = model_graph.getSorted(); if (sorted.empty()) { ml_loge("Empty model"); @@ -886,7 +889,7 @@ void NeuralNetwork::print(std::ostream &out, unsigned int flags, } // TODO: get sorted layers if initialized - auto &layers = model_graph.getLayers(); + auto layers = model_graph.getLayers(); if (flags & PRINT_GRAPH_INFO) { out << "graph contains " << layers.size() << " operation nodes\n"; /// @todo print graph info diff --git a/test/unittest/unittest_nntrainer_models.cpp b/test/unittest/unittest_nntrainer_models.cpp index 8a7f9bb6..c3833c3d 100644 --- a/test/unittest/unittest_nntrainer_models.cpp +++ b/test/unittest/unittest_nntrainer_models.cpp @@ -791,6 +791,7 @@ TEST(nntrainerModels, read_save_01_n) { {"input_shape=1:1:62720", "normalization=true", "bias_initializer=zeros"}); EXPECT_NO_THROW(NN.addLayer(layer)); + EXPECT_NO_THROW(NN.setProperty({"loss=mse"})); EXPECT_THROW(NN.readModel(), std::runtime_error); EXPECT_THROW(NN.saveModel(), std::runtime_error); -- 2.34.1