private:
void makeRank(const graph::Graph &model);
+ /**
+ * @brief Returns the time, when backend is available for at least given amount of time.
+ *
+ * @note Returns either hole/gap between two performing two already scheduled operations,
+ * or the finishing time of the last scheduled operation
+ *
+ * @param[in] backend backend, for which to return the time
+ * @param[in] starting_time time, starting which to look for gap
+ * @param[in] time_amount amound of the time, for which to look gap
+ */
+ int64_t backendAvailableTime(const backend::Backend *backend, const int64_t &starting_time,
+ const int64_t &time_amount);
int64_t getTime(const backend::Backend *backend, const std::string &operation, bool quant,
uint32_t size);
private:
std::unordered_map<const backend::Backend *, std::unordered_map<std::string, int>> _run_cache;
- std::unordered_map<const backend::Backend *, int64_t> _backends_avail_time;
+ // Finishing and starting time of each backend
+ std::unordered_map<const backend::Backend *, std::map<int64_t, int64_t>> _backends_avail_time;
std::unordered_map<model::OperationIndex, int64_t> _parents_eft;
std::multimap<int64_t, model::OperationIndex, std::greater<int64_t>> _ranks;
std::shared_ptr<OpRanks> _indexed_ranks;
const auto all_backends = _backend_resolver->getAllBackends();
for (const auto *backend : all_backends)
{
- _backends_avail_time.emplace(backend, 0);
+ _backends_avail_time.emplace(backend, std::map<int64_t, int64_t>{{0, 0}});
}
// for each task select the backend with the smallest earliest finishing time(eft)
{
// Permute layer is used for data transfer and it is implemented on cpu backend
assert(_backend_resolver->getDefaultBackend()->config()->id() == "cpu");
- transfer_cost = 100; // makes the scheduler prefer keeping computations on one backend
- _run_cache[backend][other_backend->config()->id()] = 100;
+ // makes the scheduler prefer keeping computations on one backend
+ transfer_cost = operand.info().total_size() / 100;
+ _run_cache[backend][other_backend->config()->id()] = transfer_cost;
}
avg_transfer_cost += transfer_cost;
}
VERBOSE(Scheduler::makeRank) << "task prioritizing finished" << std::endl;
}
+int64_t Scheduler::backendAvailableTime(const backend::Backend *backend,
+ const int64_t &starting_time, const int64_t &time_amount)
+{
+ auto backend_times = _backends_avail_time.at(backend);
+ // finishing and starting times of an op, that will come after current op
+ auto next_op_fst = backend_times.upper_bound(starting_time);
+ // finishing time of an op, that will come before current op
+ auto prev_op_ft = starting_time;
+ // until reach the "hole/gap", that is enough to run this op
+ while (next_op_fst != backend_times.end() && next_op_fst->second - prev_op_ft < time_amount)
+ {
+ prev_op_ft = next_op_fst->first;
+ ++next_op_fst;
+ }
+ return prev_op_ft;
+}
+
void Scheduler::scheduleNode(const graph::Graph &model, const model::OperationIndex &index)
{
VERBOSE(Scheduler::scheduleNode) << "scheduling (" << index.value() << ")" << std::endl;
const auto all_backends = _backend_resolver->getAllBackends();
- int64_t eft = std::numeric_limits<int64_t>::max();
+ int64_t eft = std::numeric_limits<int64_t>::max(), selected_exec_time = 0;
const auto &node = model.operations().at(index);
const auto &obj = model.operands().at(node.getInputs().at(0));
bool quant = false;
}
auto size = getOperationsFlattenedIOSize(model, node);
-
+ std::vector<std::pair<int64_t, int64_t>> selected_permute_avail_time;
// select the backend with the smallest eft of this task
const backend::Backend *chosen_backend = nullptr;
for (const auto *backend : all_backends)
{
+ std::vector<std::pair<int64_t, int64_t>> permute_avail_time;
+
/* TODO: this is workaround, come up with better solution if have.
Adding exception in stage doesn't help. Because if there is a record for add without
broadcast, scheduling will select it since it doesn't distinguish broadcast and
}
// get average exec time of the op on this backend
auto exec_time = getTime(backend, node.getName(), quant, size);
- VERBOSE(Scheduler::scheduleNode) << "expected exec_time of (" << index.value() << ") "
- << node.getName() << " quant==" << quant << " on "
- << backend->config()->id() << " is " << exec_time
- << " microseconds" << std::endl;
// get max eft of direct (one lavel above) predecessors
int64_t max_pred_eft = 0;
+ int64_t total_transfer_cost = 0;
for (const auto &input : node.getInputs())
{
const auto &operand = model.operands().at(input);
// Data transfer cost from this parent's backend to current node's backend:
auto parent_backend = _backend_resolver->getBackend(defs);
- int64_t transfer_cost;
if (parent_backend == backend)
{
- transfer_cost = 1;
+ max_pred_eft = std::max(max_pred_eft, _parents_eft.at(defs));
}
else
{
- transfer_cost =
+ int64_t transfer_cost =
getTime(parent_backend, backend->config()->id(), quant, obj.info().total_size());
+ total_transfer_cost += transfer_cost;
+
+ // Assumes that cpu is free and proceeds to permutation right away
+ permute_avail_time.push_back({transfer_cost, _parents_eft.at(defs)});
+ max_pred_eft = std::max(max_pred_eft, _parents_eft.at(defs));
}
- max_pred_eft = std::max(max_pred_eft, _parents_eft.at(defs) + transfer_cost);
}
}
- auto backend_avail_time = _backends_avail_time[backend];
- // earliest starting time
- auto est = std::max(backend_avail_time, max_pred_eft);
- if (eft > est + exec_time)
+ max_pred_eft += total_transfer_cost;
+ // find the hole/gap, where this op can be put or the finishing time of the last assigned op
+ auto prev_op_ft = backendAvailableTime(backend, max_pred_eft, exec_time);
+
+ VERBOSE(Scheduler::scheduleNode)
+ << "expected exec_time of (" << index.value() << ") " << node.getName()
+ << " quant==" << quant << " on " << backend->config()->id() << " is " << exec_time
+ << " microseconds. Backend available time: " << prev_op_ft
+ << " Parent's max eft: " << max_pred_eft - total_transfer_cost
+ << " total transfer cost: " << total_transfer_cost << std::endl;
+ if (eft > prev_op_ft + exec_time)
{
- eft = est + exec_time;
+ eft = prev_op_ft + exec_time;
+ selected_exec_time = exec_time;
chosen_backend = backend;
+ selected_permute_avail_time = permute_avail_time;
}
}
+ for (const auto &it : selected_permute_avail_time)
+ {
+ auto prev_op_ft =
+ backendAvailableTime(_backend_resolver->getBackend("cpu"), it.first, it.second);
+ _backends_avail_time[_backend_resolver->getBackend("cpu")].insert(
+ {prev_op_ft + it.second, prev_op_ft});
+ }
VERBOSE(Scheduler::schedule) << "backend for " << node.getName() << " is "
<< chosen_backend->config()->id() << ". Its eft: " << eft
<< std::endl;
_parents_eft[index] = eft;
- _backends_avail_time[chosen_backend] = eft + 1;
+ _backends_avail_time[chosen_backend].insert({eft, eft - selected_exec_time});
_backend_resolver->setBackend(index, chosen_backend);
}