From 662d87b2c8dd40eb0bb81888c0b6b715d136df88 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Mon, 25 Jan 2021 15:56:27 +0900 Subject: [PATCH] [graph] In-place bug fix This patch applied bug fix for in-place layer optimization. As in-place layers and other layers manages derivative memory differently, in-place layers cannot directly re-use the tensors of their neighboring layers. But rather have to use tensors differently. This patch applies this bug-fix. See also #878 **Self evaluation:** 1. Build test: [x]Passed [ ]Failed [ ]Skipped 2. Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Parichay Kapoor --- nntrainer/graph/network_graph.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp index 0503028..c237c34 100644 --- a/nntrainer/graph/network_graph.cpp +++ b/nntrainer/graph/network_graph.cpp @@ -576,8 +576,22 @@ void NetworkGraph::inPlaceOptimize(const std::string &layer_type, continue; /** Share tensor with next layer */ - l->net_input[0] = l->net_hidden[0]; - prev_layer->net_hidden[loc] = l->net_hidden[0]; + l->net_input[0] = l->net_hidden[0]; // I2 = O2 + if (l->getType() == BatchNormalizationLayer::type) { + prev_layer->net_hidden[loc] = l->net_hidden[0]; // O1 = O2 + } else if (l->getType() == ActivationLayer::type) { + // THIS IS NOT WORKING BECAUSE INITIALIZES MAKES NEW SHARED_PTR OF TENSOR + // THAN KEEPING THE SAME SHARED_PTR AND JUST CHANGING THE TENSOR IN IT + prev_layer->net_hidden[loc]->getGradientRef() = + l->net_hidden[0]->getVariableRef(); // O1.G = O2.V + prev_layer->net_hidden[loc]->getVariableRef() = + l->net_hidden[0]->getVariableRef(); // O1.V = O2.V + } + + /** prev layer should use this shared output for derivative though */ + // prev_layer->net_hidden[loc] = + // std::shared_ptr(*prev_layer->net_hidden[loc].get()); + // prev_layer->net_hidden[loc].get /** Untrack the memory for this layer */ manager.untrackLayerInOuts(prev_layer->getName()); -- 2.7.4