From b6e2cd692bfb5db32f40a13554d1474e9f751f0b Mon Sep 17 00:00:00 2001
From: Krzysztof Bruniecki <krzysztof.bruniecki@intel.com>
Date: Tue, 10 Nov 2020 11:40:28 +0100
Subject: [PATCH] Add fix for multiple_activations test (#2984)

* Add fix for multiple_activations test

   - Add forbid activation fusing pass for GNA2 library
   - Fix get all prev layers fnct
   - To enable GNA_NoRegression.smoke_multiple_activations_onGNA_INT16

* Apply Bartek review
---
 .../src/gna_plugin/gna_graph_tools.hpp             |  5 ++++
 inference-engine/src/gna_plugin/gna_plugin.cpp     |  3 +++
 .../src/gna_plugin/optimizer/gna_pass_manager.cpp  | 31 ++++++++++++++++++++++
 .../src/gna_plugin/optimizer/gna_pass_manager.hpp  |  8 ++++++
 4 files changed, 47 insertions(+)

diff --git a/inference-engine/src/gna_plugin/gna_graph_tools.hpp b/inference-engine/src/gna_plugin/gna_graph_tools.hpp
index 741951f..137543b 100644
--- a/inference-engine/src/gna_plugin/gna_graph_tools.hpp
+++ b/inference-engine/src/gna_plugin/gna_graph_tools.hpp
@@ -220,9 +220,14 @@ inline std::pair<InferenceEngine::CNNLayerPtr, int>  CNNNetCheckNextLayerSkipCer
             separate_layers(getInputTo(layer->outData[i]));
         }
 
+        std::set< CNNLayerPtr > visited;
         while (!currentSet.empty()) {
             auto currentLayer = currentSet.front();
             currentSet.pop_front();
+            if (visited.count(currentLayer)) {
+                continue;
+            }
+            visited.insert(currentLayer);
             for (auto && oData : currentLayer->outData) {
                 separate_layers(getInputTo(oData));
             }
diff --git a/inference-engine/src/gna_plugin/gna_plugin.cpp b/inference-engine/src/gna_plugin/gna_plugin.cpp
index 69ae31c..5f0a04f 100644
--- a/inference-engine/src/gna_plugin/gna_plugin.cpp
+++ b/inference-engine/src/gna_plugin/gna_plugin.cpp
@@ -421,6 +421,9 @@ void GNAPlugin::LoadNetwork(ICNNNetwork & _network) {
         passes->registerPass<InsertCopyLayerPass>();
         passes->registerPass<InsertDiagonalLayerPass>();
         passes->registerPass<HandleMultipleActivationsForTheLayerPass>();
+#if GNA_LIB_VER == 2
+        passes->registerPass<ForbidActivationFusingPass>();
+#endif
         passes->registerPass<SubstituteScaleShiftBroadCastPass>();
         passes->registerPass<FuseMultipleIdentitiesPass>();
         passes->registerPass<BroadcastConstPass>();
diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp
index 0825a52..3bae254 100644
--- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp
+++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.cpp
@@ -26,6 +26,7 @@
 #include <layers/gna_copy_layer.hpp>
 
 #include "gna_plugin_log.hpp"
+#include "frontend/quantization.h"
 #include "frontend/quantized_layer_params.hpp"
 #include <layers/gna_copy_layer.hpp>
 #include "gna_graph_tools.hpp"
@@ -268,6 +269,36 @@ void HandleMultipleActivationsForTheLayerPass::run() {
     }
 }
 
+void ForbidActivationFusingPass::run() {
+    for (auto& l : *pLayers) {
+        if (LayerInfo(l).isActivation()) {
+            auto prevLayer = CNNNetPrevLayer(l);
+            if (LayerInfo(prevLayer).has32BOutput()) {
+                // find all layers directly connected to the outputs of the previous layer
+                const auto allUsingPrev = CNNNetGetAllNextLayersSkipCertain(prevLayer, -1,
+                    [&](CNNLayerPtr nextLayer) -> bool {
+                        for (const auto& input : nextLayer->insData) {
+                            for (const auto& output : prevLayer->outData) {
+                                if (areEqualDatas(input.lock(), output) &&
+                                    areEqualDatas(l->insData[0].lock(), output) &&
+                                    (LayerInfo(nextLayer).isEltwiseSum() || nextLayer == l)) {
+                                    return false;
+                                }
+                            }
+                        }
+                        return true;
+                    });
+                if (allUsingPrev.size() > 1) {
+                    // the weights of MAX_VAL_2B_WEIGHT are used to enforce 1.0 scale factor
+                    // so the scores are more correct
+                    insertDiagonalLayerBetween(prevLayer, l, getPassManager(), MAX_VAL_2B_WEIGHT);
+                }
+                continue;
+            }
+        }
+    }
+}
+
 void ReorderMaxPoolPass::run() {
     // detecting following pattern
     // conv->relu->maxpooling
diff --git a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp
index 7e2957a..6ee8b5c 100644
--- a/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp
+++ b/inference-engine/src/gna_plugin/optimizer/gna_pass_manager.hpp
@@ -124,6 +124,14 @@ DECL_PASS(ReorderMaxPool);
 DECL_PASS(HandleMultipleActivationsForTheLayer);
 
 /**
+ * @brief GNA doesn't provide intermediate results (sums) when the layer is fused with activation.
+ * When more layers use the sums as inputs (beside the activation) then the diagonal layer
+ * is inserted before the activation to forbid the fusing and make the sums exposed.
+ * This is observed in the multiple_activations_onGNA_INT16 test.
+ */
+DECL_PASS(ForbidActivationFusing);
+
+/**
  * @brief copy layer insertion required in cases where input layer does not have output memory
  */
 DECL_PASS(InsertCopyLayer);
-- 
2.7.4