[libbenchmark] Introduce Result class (#9364)

Data class between runners(nnpackage_run and tflite_run) and libbenchmark

Signed-off-by: Yongseop Kim <yons.kim@samsung.com>

diff --git a/runtime/libs/benchmark/include/benchmark.h b/runtime/libs/benchmark/include/benchmark.h
index 3445700..cd1298a 100644 (file)
--- a/runtime/libs/benchmark/include/benchmark.h
+++ b/runtime/libs/benchmark/include/benchmark.h
@@ -17,6 +17,7 @@
 #ifndef __NNFW_BENCHMARK_H__
 #define __NNFW_BENCHMARK_H__
 
+#include "benchmark/Result.h"
 #include "benchmark/MemoryPoller.h"
 #include "benchmark/CsvWriter.h"
 

diff --git a/runtime/libs/benchmark/include/benchmark/MemoryPoller.h b/runtime/libs/benchmark/include/benchmark/MemoryPoller.h
index 8d2f61e..b3570c5 100644 (file)
--- a/runtime/libs/benchmark/include/benchmark/MemoryPoller.h
+++ b/runtime/libs/benchmark/include/benchmark/MemoryPoller.h
@@ -30,6 +30,7 @@ namespace benchmark
 {
 
 // NOTE. gpu_poll is not necessary on general targets. This is used on the only tv targets.
+// TODO finally should be separated from data
 class MemoryPoller
 {
 public:
@@ -46,6 +47,8 @@ public:
   bool Start(const std::string &phase);
   uint32_t End(const std::string &phase);
 
+  // TODO expose rss & hwm data
+
 private:
   void Process();
   bool PrepareMemoryPolling();

diff --git a/runtime/libs/benchmark/include/benchmark/Result.h b/runtime/libs/benchmark/include/benchmark/Result.h
new file mode 100644 (file)
index 0000000..253b336
--- /dev/null
+++ b/runtime/libs/benchmark/include/benchmark/Result.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __NNFW_BENCHMARK_RESULT_H__
+#define __NNFW_BENCHMARK_RESULT_H__
+
+#include <string>
+#include <vector>
+#include <unordered_map>
+#include <numeric>
+#include <algorithm>
+#include <cassert>
+
+namespace
+{
+
+uint32_t MaxMemory(const std::unordered_map<std::string, uint32_t> &map)
+{
+  auto answer =
+      *std::max_element(map.begin(), map.end(), [](const std::pair<std::string, uint32_t> &p1,
+                                                   const std::pair<std::string, uint32_t> &p2) {
+        return p1.second < p2.second;
+      });
+  return answer.second;
+}
+
+} // namespace anonymous
+
+namespace benchmark
+{
+
+// Data class between runner(nnpackage_run and tflite_run) and libbenchmark
+class Result
+{
+public:
+  Result(const std::string &model, const std::string &backend, double model_load_time,
+         double prepare_time, const std::vector<double> &execute_time,
+         const std::unordered_map<std::string, uint32_t> &rss,
+         const std::unordered_map<std::string, uint32_t> &hwm)
+      : _model(model), _backend(backend), _model_load_time(model_load_time),
+        _prepare_time(prepare_time)
+  {
+    // execute
+    double sum = 0.0;
+    double min = std::numeric_limits<double>::max();
+    double max = std::numeric_limits<double>::lowest();
+    for (auto t : execute_time)
+    {
+      sum += t;
+      min = std::min(min, t);
+      max = std::max(max, t);
+    }
+    _execute_time_mean = sum / static_cast<double>(execute_time.size());
+    _execute_time_min = min;
+    _execute_time_max = max;
+
+    // rss
+    assert(rss.size() > 0);
+    assert(rss.find("Compiling") != rss.end());
+    assert(rss.find("Executing") != rss.end());
+    // _model_load_rss = rss.at(Phase::MODEL_LOAD);
+    _prepare_rss = rss.at("Compiling");
+    _execute_rss = rss.at("Executing");
+    _peak_rss = MaxMemory(rss);
+
+    // hwm
+    assert(hwm.size() > 0);
+    assert(hwm.find("Compiling") != hwm.end());
+    assert(hwm.find("Executing") != hwm.end());
+    // _model_load_hwm = hwm.at(Phase::MODEL_LOAD);
+    _prepare_hwm = hwm.at("Compiling");
+    _execute_hwm = hwm.at("Executing");
+    _peak_hwm = MaxMemory(hwm);
+  }
+
+public:
+  const std::string &model() { return _model; }
+  const std::string &backend() { return _backend; }
+  const std::string &model() const { return _model; }
+  const std::string &backend() const { return _backend; }
+  double model_load_time() { return _model_load_time; }
+  double prepare_time() { return _prepare_time; }
+  double execute_time_mean() { return _execute_time_mean; }
+  double execute_time_min() { return _execute_time_min; }
+  double execute_time_max() { return _execute_time_max; }
+  uint32_t model_load_rss() { return _model_load_rss; }
+  uint32_t prepare_rss() { return _prepare_rss; }
+  uint32_t execute_rss() { return _execute_rss; }
+  uint32_t peak_rss() { return _peak_rss; }
+  uint32_t model_load_hwm() { return _model_load_hwm; }
+  uint32_t prepare_hwm() { return _prepare_hwm; }
+  uint32_t execute_hwm() { return _execute_hwm; }
+  uint32_t peak_hwm() { return _peak_hwm; }
+
+private:
+  std::string _model;
+  std::string _backend;
+  double _model_load_time;
+  double _prepare_time;
+  double _execute_time_mean;
+  double _execute_time_min;
+  double _execute_time_max;
+  uint32_t _model_load_rss;
+  uint32_t _prepare_rss;
+  uint32_t _execute_rss;
+  uint32_t _peak_rss;
+  uint32_t _model_load_hwm;
+  uint32_t _prepare_hwm;
+  uint32_t _execute_hwm;
+  uint32_t _peak_hwm;
+};
+
+} // namespace benchmark
+
+#endif // __NNFW_BENCHMARK_RESULT_H__

diff --git a/tests/tools/nnpackage_run/src/nnpackage_run.cc b/tests/tools/nnpackage_run/src/nnpackage_run.cc
index 3bf2d38..f023e3a 100644 (file)
--- a/tests/tools/nnpackage_run/src/nnpackage_run.cc
+++ b/tests/tools/nnpackage_run/src/nnpackage_run.cc
@@ -41,7 +41,7 @@ template <class T> void randomData(RandomGenerator &randgen, void *data, uint64_
 }
 }
 
-// TODO Replace this with nnfw::misc::benchmark::Accumulator
+// TODO Remove when unnecessary
 namespace benchmark
 {
 

diff --git a/tests/tools/tflite_run/src/tflite_run.cc b/tests/tools/tflite_run/src/tflite_run.cc
index ddf01dd..174896e 100644 (file)
--- a/tests/tools/tflite_run/src/tflite_run.cc
+++ b/tests/tools/tflite_run/src/tflite_run.cc
@@ -42,7 +42,7 @@ using namespace tflite;
 using namespace nnfw::tflite;
 using namespace std::placeholders; // for _1, _2 ...
 
-// TODO Replace this with nnfw::misc::benchmark::Accumulator
+// TODO Remove when unnecessary
 namespace benchmark
 {