From 482d4efc2bbc867dd7ed0cef30d05ea8badfba1d Mon Sep 17 00:00:00 2001 From: =?utf8?q?=EB=B0=95=EC=A2=85=ED=98=84/=EB=8F=99=EC=9E=91=EC=A0=9C?= =?utf8?q?=EC=96=B4Lab=28SR=29/Senior=20Engineer/=EC=82=BC=EC=84=B1?= =?utf8?q?=EC=A0=84=EC=9E=90?= Date: Wed, 18 Apr 2018 07:16:26 +0900 Subject: [PATCH] Show max absolute/relative diff (#761) This commit revises NNAPI test framework to show max absolute/relative diff information at the end. Signed-off-by: Jonghyun Park --- include/support/tflite/Diff.h | 10 ++++- src/support/tflite/src/Diff.cpp | 96 +++++++++++++++++++++++++++++++---------- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/include/support/tflite/Diff.h b/include/support/tflite/Diff.h index 63d78e4..46f1d5e 100644 --- a/include/support/tflite/Diff.h +++ b/include/support/tflite/Diff.h @@ -17,7 +17,6 @@ struct TfLiteTensorDiff nnfw::util::tensor::Index index; float expected; float obtained; - float relative_diff; TfLiteTensorDiff(const nnfw::util::tensor::Index &i) : index(i) { @@ -34,9 +33,16 @@ public: } public: + struct Observer + { + virtual void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) = 0; + }; + +public: + // NOTE Observer should live longer than comparator std::vector compare(const nnfw::support::tflite::TensorView &expected, const nnfw::support::tflite::TensorView &obtained, - TfLiteTensorDiff *max_diff = nullptr) const; + Observer *observer = nullptr) const; private: std::function _compare_fn; diff --git a/src/support/tflite/src/Diff.cpp b/src/support/tflite/src/Diff.cpp index dd79270..01cc25e 100644 --- a/src/support/tflite/src/Diff.cpp +++ b/src/support/tflite/src/Diff.cpp @@ -8,10 +8,57 @@ #include +class DiffSummary : public TfLiteTensorComparator::Observer +{ +public: + DiffSummary() + : max_abs_diff_index(0), max_abs_diff_value{0.0f}, + max_rel_diff_index(0), max_rel_diff_value{0.0f} + { + // DO NOTHING + } +public: + void notify(const nnfw::util::tensor::Index &index, float expected, float obtained) override; + +public: + nnfw::util::tensor::Index max_abs_diff_index; + float max_abs_diff_expected; + float max_abs_diff_obtained; + float max_abs_diff_value; + + nnfw::util::tensor::Index max_rel_diff_index; + float max_rel_diff_expected; + float max_rel_diff_obtained; + float max_rel_diff_value; +}; + +void DiffSummary::notify(const nnfw::util::tensor::Index &index, float expected, float obtained) +{ + const auto abs_diff_value = std::fabs(expected - obtained); + + if (max_abs_diff_value < abs_diff_value) + { + max_abs_diff_index = index; + max_abs_diff_value = abs_diff_value; + max_abs_diff_expected = expected; + max_abs_diff_obtained = obtained; + } + + const auto rel_diff_value = nnfw::util::fp32::relative_diff(expected, obtained); + + if (max_rel_diff_value < rel_diff_value) + { + max_rel_diff_index = index; + max_rel_diff_value = rel_diff_value; + max_rel_diff_expected = expected; + max_rel_diff_obtained = obtained; + } +} + std::vector TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView &expected, const nnfw::support::tflite::TensorView &obtained, - TfLiteTensorDiff *max_diff) const + Observer *observer) const { std::vector res; @@ -28,21 +75,14 @@ TfLiteTensorComparator::compare(const nnfw::support::tflite::TensorView & diff.expected = expected_value; diff.obtained = obtained_value; - diff.relative_diff = relative_diff; res.emplace_back(diff); } // Update max_diff_index, if necessary - if (max_diff != nullptr) + if (observer != nullptr) { - if (max_diff->relative_diff < relative_diff) - { - max_diff->index = index; - max_diff->expected = expected_value; - max_diff->obtained = obtained_value; - max_diff->relative_diff = relative_diff; - } + observer->notify(index, expected_value, obtained_value); } }; @@ -58,10 +98,9 @@ bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpre const auto expected = nnfw::support::tflite::TensorView::make(interp, id); const auto obtained = nnfw::support::tflite::TensorView::make(nnapi, id); - TfLiteTensorDiff max_diff(0); - max_diff.relative_diff = 0; + DiffSummary summary; - auto diffs = _comparator.compare(expected, obtained, &max_diff); + auto diffs = _comparator.compare(expected, obtained, &summary); if (diffs.size() == 0) { @@ -70,19 +109,26 @@ bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpre else { std::cout << " Tensor #" << id << ": UNMATCHED" << std::endl; + std::cout << " " << diffs.size() << " diffs are detected" << std::endl; } // Print out max_diff - if (max_diff.relative_diff > 0) + if (summary.max_abs_diff_value > 0) { - const auto tolerance_level = max_diff.relative_diff / FLT_EPSILON; + std::cout << " Max absolute diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl; + std::cout << " expected: " << summary.max_abs_diff_expected << std::endl; + std::cout << " obtained: " << summary.max_abs_diff_obtained << std::endl; + std::cout << " absolute diff: " << summary.max_abs_diff_value << std::endl; + } - std::cout << " " << diffs.size() << " diffs are detected" << std::endl; - std::cout << " Max at [" << nnfw::util::tensor::IndexFormatter(max_diff.index) << "]" << std::endl; - std::cout << " expected: " << max_diff.expected << std::endl; - std::cout << " obtained: " << max_diff.obtained << std::endl; - std::cout << " absolute diff: " << std::fabs(max_diff.expected - max_diff.obtained) << std::endl; - std::cout << " relative diff: " << max_diff.relative_diff << std::endl; + if (summary.max_rel_diff_value > 0) + { + const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON; + + std::cout << " Max relative diff at [" << nnfw::util::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl; + std::cout << " expected: " << summary.max_rel_diff_expected << std::endl; + std::cout << " obtained: " << summary.max_rel_diff_obtained << std::endl; + std::cout << " relative diff: " << summary.max_rel_diff_value << std::endl; std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl; } @@ -93,10 +139,16 @@ bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpre std::cout << " ---- Details ---" << std::endl; for (const auto &diff : diffs) { + const auto absolute_diff = std::fabs(diff.expected - diff.obtained); + const auto relative_diff = nnfw::util::fp32::relative_diff(diff.expected, diff.obtained); + const auto tolerance_level = relative_diff / FLT_EPSILON; + std::cout << " Diff at [" << nnfw::util::tensor::IndexFormatter(diff.index) << "]" << std::endl; std::cout << " expected: " << diff.expected << std::endl; std::cout << " obtained: " << diff.obtained << std::endl; - std::cout << " relative diff: " << diff.relative_diff << std::endl; + std::cout << " absolute diff: " << absolute_diff << std::endl; + std::cout << " relative diff: " << relative_diff << std::endl; + std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl; } } -- 2.7.4