[neurun] Introduce PermutationInsertionPass (#2861)

Introduce PermutationInsertionPass that enables to run the model with
heterogeneous backends. This commit also applies this pass in
`Graph::lower`.

Signed-off-by: Hanjoung Lee <hanjoung.lee@samsung.com>

diff --git a/runtimes/neurun/src/graph/Graph.cc b/runtimes/neurun/src/graph/Graph.cc
index 069ee07..3d03127 100644 (file)
--- a/runtimes/neurun/src/graph/Graph.cc
+++ b/runtimes/neurun/src/graph/Graph.cc
@@ -29,6 +29,7 @@
 #include "codegen/BackendResolver.h"
 #include "backend/IBackendConfig.h"
 #include "operation/Permute.h"
+#include "pass/PermutationInsertionPass.h"
 
 namespace neurun
 {
@@ -221,13 +222,19 @@ void Graph::lower(void)
     });
   }
 
+  _phase = Phase::LOWERED;
+
+  // Run PermutationInsertionPass
+  {
+    pass::PermutationInsertionPass pi_pass(*this);
+    pi_pass.run();
+  }
+
   // Graph verifications for the LOWERED phase
   {
     verifier::DAGChecker dag_checker;
     dag_checker.verify(*this);
   }
-
-  _phase = Phase::LOWERED;
 }
 
 std::unique_ptr<linear::Linear> Graph::linearize(void)

diff --git a/runtimes/neurun/src/graph/Graph.h b/runtimes/neurun/src/graph/Graph.h
index 7368fa3..304b9ea 100644 (file)
--- a/runtimes/neurun/src/graph/Graph.h
+++ b/runtimes/neurun/src/graph/Graph.h
@@ -122,6 +122,7 @@ public:
   const operand::Set &operands() const { return _operands; }
   operand::Set &operands() { return _operands; } // TODO Remove this non-const accessor
   const operation::Set &operations() const { return _operations; }
+  operation::Set &operations() { return _operations; }
 
 private:
   Phase _phase{Phase::BUILDING};

diff --git a/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc b/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc
new file mode 100644 (file)
index 0000000..703072d
--- /dev/null
+++ b/runtimes/neurun/src/graph/pass/PermutationInsertionPass.cc
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "PermutationInsertionPass.h"
+
+#include <cassert>
+#include <utility>
+#include <unordered_map>
+
+#include "graph/operand/Object.h"
+#include "graph/operation/LowerInfo.h"
+#include "graph/Graph.h"
+#include "backend/IBackendConfig.h"
+#include "logging.h"
+
+namespace neurun
+{
+namespace graph
+{
+namespace pass
+{
+
+void PermutationInsertionPass::callback(const operand::Index &index, operand::Object &object)
+{
+  auto &&operand_li = object.lower_info();
+  assert(operand_li);
+
+  // NOTE Later, constants also will have Def
+  // Ignore constants
+  if (operand_li->def_backends().size() == 0)
+  {
+    return;
+  }
+
+  std::list<operation::Index> permute_indexes;
+
+  // Build a map for all necessary type of operands
+  std::unordered_map<const backend::Backend *, operand::Index> backend_to_index;
+  {
+    assert(operand_li->def_backends().size() == 1);
+    for (auto backend : operand_li->def_backends())
+    {
+      backend_to_index.insert({backend, index});
+    }
+
+    auto insert_set = operand_li->use_backends() - operand_li->def_backends();
+    for (auto backend : insert_set)
+    {
+      const auto permute_operation_index = _graph.insertPermute(index, backend);
+      permute_indexes.push_back(permute_operation_index);
+      VERBOSE(PermutationInsertionPass) << "Insert 'Permute' operation for operand "
+                                        << index.value() << std::endl;
+      const auto &permute_operation = _graph.operations().at(permute_operation_index);
+      const auto permuted_operand_index = permute_operation.getOutputs().at(0);
+      backend_to_index.insert({backend, permuted_operand_index});
+    }
+  }
+
+  // Update operations' input that uses this operand
+  {
+    std::list<operation::Index> remove_list;
+
+    auto uses = object.getUses();
+    for (auto use : uses.list())
+    {
+      // If permute operation, ignore it
+      if (std::find(permute_indexes.begin(), permute_indexes.end(), use) != permute_indexes.end())
+        continue;
+
+      auto &operation = _graph.operations().at(use);
+      auto operation_li = operation.lower_info();
+      assert(operation_li);
+      auto backend = operation_li->backend();
+
+      auto use_node_inputs = operation.getInputs();
+      assert(use_node_inputs.contains(index));
+
+      auto new_index = backend_to_index.at(backend);
+      if (index != new_index)
+      {
+        // Update from operation
+        operation.replaceInput(index, new_index);
+
+        // Update from operand
+        remove_list.push_back(
+            use); // Removal should be done in another loop since we are in the loop
+        _graph.operands().at(new_index).appendUse(use);
+      }
+    }
+
+    for (auto &operation : remove_list)
+    {
+      object.removeUse(operation);
+    }
+  }
+}
+
+} // namespace pass
+} // namespace graph
+} // namespace neurun

diff --git a/runtimes/neurun/src/graph/pass/PermutationInsertionPass.h b/runtimes/neurun/src/graph/pass/PermutationInsertionPass.h
new file mode 100644 (file)
index 0000000..1654f68
--- /dev/null
+++ b/runtimes/neurun/src/graph/pass/PermutationInsertionPass.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __NEURUN_GRAPH_PASS_PERMUTATION_INSERTION_PASS_H__
+#define __NEURUN_GRAPH_PASS_PERMUTATION_INSERTION_PASS_H__
+
+#include "OperandPass.h"
+
+namespace neurun
+{
+namespace graph
+{
+namespace pass
+{
+
+class PermutationInsertionPass : public OperandPass
+{
+public:
+  using OperandPass::OperandPass;
+
+public:
+  virtual std::string id() override { return "PermutationInsertionPass"; }
+  virtual void callback(const operand::Index &index, operand::Object &object);
+
+private:
+};
+
+} // namespace pass
+} // namespace graph
+} // namespace neurun
+
+#endif // __NEURUN_GRAPH_PASS_PERMUTATION_INSERTION_PASS_H__