[neurun] Elimination data transfer assumption in Scheduler (#5818)

Eliminate multiplying data tranfer time by 2 and calculate
free time slot for permutation

Signed-off-by: Dilshodzhon Poshshoev <d.poshshoev@samsung.com>

diff --git a/runtimes/neurun/core/src/compiler/Scheduler.cc b/runtimes/neurun/core/src/compiler/Scheduler.cc
index 53a2f24..13f23db 100644 (file)
--- a/runtimes/neurun/core/src/compiler/Scheduler.cc
+++ b/runtimes/neurun/core/src/compiler/Scheduler.cc
@@ -229,6 +229,7 @@ int64_t Scheduler::tryBackend(const model::Operation &node, const backend::Backe
   return _run_cache[backend][node.getName()];
 }
 
+// TODO: Too large function: need code refactoring
 void Scheduler::makeRank(const graph::Graph &model)
 {
   VERBOSE(Scheduler::makeRank) << "task prioritizing" << std::endl;
@@ -365,6 +366,7 @@ int64_t Scheduler::backendAvailableTime(const backend::Backend *backend,
   return prev_op_ft;
 }
 
+// TODO: Too large function: need code refactoring
 void Scheduler::scheduleNode(const graph::Graph &model, const model::OperationIndex &index)
 {
   const bool is_linear_exec = "Linear" == util::getConfigString(util::config::EXECUTOR);
@@ -380,12 +382,12 @@ void Scheduler::scheduleNode(const graph::Graph &model, const model::OperationIn
   bool quant = isQuant(model, node);
 
   auto size = getOperationsFlattenedIOSize(model, node);
-  std::vector<std::pair<int64_t, int64_t>> selected_permute_avail_time;
+  std::multimap<int64_t, int64_t> selected_permute_avail_time;
   // select the backend with the smallest eft of this task
   const backend::Backend *chosen_backend = nullptr;
   for (const auto *backend : all_backends)
   {
-    std::vector<std::pair<int64_t, int64_t>> permute_avail_time;
+    std::multimap<int64_t, int64_t> permute_avail_time;
     if (isWorkaroundSkip(model, backend, node, quant))
     {
       continue;
@@ -419,25 +421,36 @@ void Scheduler::scheduleNode(const graph::Graph &model, const model::OperationIn
               getTime(parent_backend, backend->config()->id(), quant, obj.info().total_size() * 2);
           if (!is_linear_exec)
           {
-            // Multiply data transfer time by 2 because in parallel executor it might be busy with
-            // permutation on other branches or non-nnfw specific tasks and have to wait for it.
-            // Number 2 is picked experimentally
-            // TODO: remove multiplying by 2. PR #5775, the first commit
-            transfer_cost *= CPU_DELAY * 2;
+            transfer_cost *= CPU_DELAY;
           }
           total_transfer_cost += transfer_cost;
-          auto prev_op_ft = backendAvailableTime(_backend_resolver->getBackend("cpu"),
-                                                 _parents_eft.at(defs), transfer_cost);
-          permute_avail_time.push_back({prev_op_ft, transfer_cost});
-          max_pred_eft = std::max(max_pred_eft, prev_op_ft);
+          permute_avail_time.insert({_parents_eft.at(defs), transfer_cost});
         }
       }
     }
 
-    max_pred_eft += total_transfer_cost;
+    std::vector<std::multimap<int64_t, int64_t>::iterator> inserted_permutations;
+    // Find free time for data transfering and insert it into backend taskset. This is needed:
+    //  1. Time for multiple permutations for this node's input is found correctly
+    //  2. If backend==cpu, then free time for this node must come after permutations
+    for (const auto &it : permute_avail_time)
+    {
+      auto prev_op_ft =
+          backendAvailableTime(_backend_resolver->getBackend("cpu"), it.first, it.second);
+      max_pred_eft = std::max(max_pred_eft, prev_op_ft + it.second);
+
+      auto tmp = _backends_avail_time[_backend_resolver->getBackend("cpu")].insert(
+          {prev_op_ft + it.second, prev_op_ft});
+      inserted_permutations.push_back(tmp.first);
+    }
     // find the hole/gap, where this op can be put or the finishing time of the last assigned op
     auto prev_op_ft = backendAvailableTime(backend, max_pred_eft, exec_time);
 
+    // Remove inserted permutation from cpu's task set
+    for (const auto &it : inserted_permutations)
+    {
+      _backends_avail_time[_backend_resolver->getBackend("cpu")].erase(it);
+    }
     // In case linear executor measure just exec time and data transfer time
     if (is_linear_exec)
     {
@@ -471,9 +484,9 @@ void Scheduler::scheduleNode(const graph::Graph &model, const model::OperationIn
     _backends_avail_time[_backend_resolver->getBackend("cpu")].insert(
         {prev_op_ft + it.second, prev_op_ft});
   }
-  VERBOSE(Scheduler::schedule) << "backend for " << node.getName() << " is "
-                               << chosen_backend->config()->id() << ". Its eft: " << eft
-                               << std::endl;
+  VERBOSE(Scheduler::scheduleNode) << "backend for " << node.getName() << " is "
+                                   << chosen_backend->config()->id() << ". Its eft: " << eft
+                                   << std::endl;
 
   _parents_eft[index] = eft;
   _backends_avail_time[chosen_backend].insert({eft, eft - selected_exec_time});