Show max absolute/relative diff (#761)

This commit revises NNAPI test framework to show max absolute/relative
diff information at the end.

Signed-off-by: Jonghyun Park <jh1302.park@samsung.com>

diff --git a/include/support/tflite/Diff.h b/include/support/tflite/Diff.h
index 63d78e4..46f1d5e 100644 (file)
--- a/include/support/tflite/Diff.h
+++ b/include/support/tflite/Diff.h
@@ -17,7 +17,6 @@ struct TfLiteTensorDiff
   nnfw::util::tensor::Index index;
   float expected;
   float obtained;
-  float relative_diff;
 
   TfLiteTensorDiff(const nnfw::util::tensor::Index &i) : index(i)
   {
@@ -34,9 +33,16 @@ public:
   }
 
 public:
+  struct Observer
+  {
+    virtual void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) = 0;
+  };
+
+public:
+  // NOTE Observer should live longer than comparator
   std::vector<TfLiteTensorDiff> compare(const nnfw::support::tflite::TensorView<float> &expected,
                                         const nnfw::support::tflite::TensorView<float> &obtained,
-                                        TfLiteTensorDiff *max_diff = nullptr) const;
+                                        Observer *observer = nullptr) const;
 
 private:
   std::function<bool (float lhs, float rhs)> _compare_fn;

diff --git a/src/support/tflite/src/Diff.cpp b/src/support/tflite/src/Diff.cpp
index dd79270..01cc25e 100644 (file)
--- a/src/support/tflite/src/Diff.cpp
+++ b/src/support/tflite/src/Diff.cpp
@@ -8,10 +8,57 @@
 
 #include <iostream>
 
+class DiffSummary : public TfLiteTensorComparator::Observer
+{
+public:
+  DiffSummary()
+      : max_abs_diff_index(0), max_abs_diff_value{0.0f},
+        max_rel_diff_index(0), max_rel_diff_value{0.0f}
+  {
+    // DO NOTHING
+  }
+public:
+  void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) override;
+
+public:
+  nnfw::util::tensor::Index max_abs_diff_index;
+  float max_abs_diff_expected;
+  float max_abs_diff_obtained;
+  float max_abs_diff_value;
+
+  nnfw::util::tensor::Index max_rel_diff_index;
+  float max_rel_diff_expected;
+  float max_rel_diff_obtained;
+  float max_rel_diff_value;
+};
+
+void DiffSummary::notify(const nnfw::util::tensor::Index &index, float expected, float obtained)
+{
+  const auto abs_diff_value = std::fabs(expected - obtained);
+
+  if (max_abs_diff_value < abs_diff_value)
+  {
+    max_abs_diff_index = index;
+    max_abs_diff_value = abs_diff_value;
+    max_abs_diff_expected = expected;
+    max_abs_diff_obtained = obtained;
+  }
+
+  const auto rel_diff_value = nnfw::util::fp32::relative_diff(expected, obtained);
+
+  if (max_rel_diff_value < rel_diff_value)
+  {
+    max_rel_diff_index = index;
+    max_rel_diff_value = rel_diff_value;
+    max_rel_diff_expected = expected;
+    max_rel_diff_obtained = obtained;
+  }
+}
+
 std::vector<TfLiteTensorDiff>
 TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView<float> &expected,
                                 const nnfw::support::tflite::TensorView<float> &obtained,
-                                TfLiteTensorDiff *max_diff) const
+                                Observer *observer) const
 {
   std::vector<TfLiteTensorDiff> res;
 
@@ -28,21 +75,14 @@ TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView<float> &
 
       diff.expected = expected_value;
       diff.obtained = obtained_value;
-      diff.relative_diff = relative_diff;
 
       res.emplace_back(diff);
     }
 
     // Update max_diff_index, if necessary
-    if (max_diff != nullptr)
+    if (observer != nullptr)
     {
-      if (max_diff->relative_diff < relative_diff)
-      {
-        max_diff->index = index;
-        max_diff->expected = expected_value;
-        max_diff->obtained = obtained_value;
-        max_diff->relative_diff =  relative_diff;
-      }
+      observer->notify(index, expected_value, obtained_value);
     }
   };
 
@@ -58,10 +98,9 @@ bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpre
     const auto expected = nnfw::support::tflite::TensorView<float>::make(interp, id);
     const auto obtained = nnfw::support::tflite::TensorView<float>::make(nnapi, id);
 
-    TfLiteTensorDiff max_diff(0);
-    max_diff.relative_diff = 0;
+    DiffSummary summary;
 
-    auto diffs = _comparator.compare(expected, obtained, &max_diff);
+    auto diffs = _comparator.compare(expected, obtained, &summary);
 
     if (diffs.size() == 0)
     {
@@ -70,19 +109,26 @@ bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpre
     else
     {
       std::cout << "  Tensor #" << id << ": UNMATCHED" << std::endl;
+      std::cout << "    " << diffs.size() << " diffs are detected" << std::endl;
     }
 
     // Print out max_diff
-    if (max_diff.relative_diff > 0)
+    if (summary.max_abs_diff_value > 0)
     {
-      const auto tolerance_level = max_diff.relative_diff / FLT_EPSILON;
+      std::cout << "    Max absolute diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl;
+      std::cout << "       expected: " << summary.max_abs_diff_expected << std::endl;
+      std::cout << "       obtained: " << summary.max_abs_diff_obtained << std::endl;
+      std::cout << "       absolute diff: " << summary.max_abs_diff_value << std::endl;
+    }
 
-      std::cout << "    " << diffs.size() << " diffs are detected" << std::endl;
-      std::cout << "    Max at [" << nnfw::util::tensor::IndexFormatter(max_diff.index) << "]" << std::endl;
-      std::cout << "       expected: " << max_diff.expected << std::endl;
-      std::cout << "       obtained: " << max_diff.obtained << std::endl;
-      std::cout << "       absolute diff: " << std::fabs(max_diff.expected - max_diff.obtained) << std::endl;
-      std::cout << "       relative diff: " << max_diff.relative_diff << std::endl;
+    if (summary.max_rel_diff_value > 0)
+    {
+      const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON;
+
+      std::cout << "    Max relative diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl;
+      std::cout << "       expected: " << summary.max_rel_diff_expected << std::endl;
+      std::cout << "       obtained: " << summary.max_rel_diff_obtained << std::endl;
+      std::cout << "       relative diff: " << summary.max_rel_diff_value << std::endl;
       std::cout << "         (tolerance level = " << tolerance_level << ")" << std::endl;
     }
 
@@ -93,10 +139,16 @@ bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpre
         std::cout << "    ---- Details ---" << std::endl;
         for (const auto &diff : diffs)
         {
+          const auto absolute_diff = std::fabs(diff.expected - diff.obtained);
+          const auto relative_diff = nnfw::util::fp32::relative_diff(diff.expected, diff.obtained);
+          const auto tolerance_level = relative_diff / FLT_EPSILON;
+
           std::cout << "    Diff at [" << nnfw::util::tensor::IndexFormatter(diff.index) << "]" << std::endl;
           std::cout << "      expected: " << diff.expected << std::endl;
           std::cout << "      obtained: " << diff.obtained << std::endl;
-          std::cout << "      relative diff: " << diff.relative_diff << std::endl;
+          std::cout << "      absolute diff: " << absolute_diff << std::endl;
+          std::cout << "      relative diff: " << relative_diff << std::endl;
+          std::cout << "         (tolerance level = " << tolerance_level << ")" << std::endl;
         }
       }