2 * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "tflite/Diff.h"
19 #include "misc/fp32.h"
21 #include "misc/tensor/IndexFormatter.h"
22 #include "misc/tensor/Zipper.h"
23 #include "misc/tensor/Comparator.h"
28 class DiffSummary : public nnfw::misc::tensor::Comparator::Observer
32 : max_abs_diff_index(0), max_abs_diff_expected{0.0f}, max_abs_diff_obtained{0.0f},
33 max_abs_diff_value{0.0f}, max_rel_diff_index(0), max_rel_diff_expected{0.0f},
34 max_rel_diff_obtained{0.0f}, max_rel_diff_value{0.0f}
40 void notify(const nnfw::misc::tensor::Index &index, float expected, float obtained) override;
43 nnfw::misc::tensor::Index max_abs_diff_index;
44 float max_abs_diff_expected;
45 float max_abs_diff_obtained;
46 float max_abs_diff_value;
48 nnfw::misc::tensor::Index max_rel_diff_index;
49 float max_rel_diff_expected;
50 float max_rel_diff_obtained;
51 float max_rel_diff_value;
54 void DiffSummary::notify(const nnfw::misc::tensor::Index &index, float expected, float obtained)
56 const auto abs_diff_value = std::fabs(expected - obtained);
58 if (max_abs_diff_value < abs_diff_value)
60 max_abs_diff_index = index;
61 max_abs_diff_value = abs_diff_value;
62 max_abs_diff_expected = expected;
63 max_abs_diff_obtained = obtained;
66 const auto rel_diff_value = nnfw::misc::fp32::relative_diff(expected, obtained);
68 if (max_rel_diff_value < rel_diff_value)
70 max_rel_diff_index = index;
71 max_rel_diff_value = rel_diff_value;
72 max_rel_diff_expected = expected;
73 max_rel_diff_obtained = obtained;
78 bool TfLiteInterpMatchApp::compareSingleTensorView(const nnfw::tflite::TensorView<T> &expected,
79 const nnfw::tflite::TensorView<T> &obtained,
82 std::vector<nnfw::misc::tensor::Diff<T>> diffs;
83 assert(expected.shape() == obtained.shape());
85 using nnfw::misc::tensor::Index;
86 using nnfw::misc::tensor::zip;
88 zip(expected.shape(), expected, obtained)
89 << [&](const Index &index, T expected_value, T obtained_value) {
90 if (expected_value != obtained_value)
92 diffs.emplace_back(index, expected_value, obtained_value);
96 // TODO Unify summary generation code
97 if (diffs.size() == 0)
99 std::cout << " Tensor #" << id << ": MATCHED" << std::endl;
103 std::cout << " Tensor #" << id << ": UNMATCHED" << std::endl;
104 std::cout << " " << diffs.size() << " diffs are detected" << std::endl;
107 if (diffs.size() > 0 && _verbose != 0)
109 std::cout << " ---- Details ---" << std::endl;
110 for (const auto &diff : diffs)
112 std::cout << " Diff at [" << nnfw::misc::tensor::IndexFormatter(diff.index) << "]"
114 std::cout << " expected: " << diff.expected << std::endl;
115 std::cout << " obtained: " << diff.obtained << std::endl;
119 return diffs.size() == 0;
123 bool TfLiteInterpMatchApp::compareSingleTensorView<float>(
124 const nnfw::tflite::TensorView<float> &expected,
125 const nnfw::tflite::TensorView<float> &obtained, int id) const
129 assert(expected.shape() == obtained.shape());
130 auto diffs = _comparator.compare(expected.shape(), expected, obtained, &summary);
132 // TODO Unify summary generation code
133 if (diffs.size() == 0)
135 std::cout << " Tensor #" << id << ": MATCHED" << std::endl;
139 std::cout << " Tensor #" << id << ": UNMATCHED" << std::endl;
140 std::cout << " " << diffs.size() << " diffs are detected" << std::endl;
143 // Print out max_diff
144 if (summary.max_abs_diff_value > 0)
146 std::cout << " Max absolute diff at ["
147 << nnfw::misc::tensor::IndexFormatter(summary.max_abs_diff_index) << "]" << std::endl;
148 std::cout << " expected: " << summary.max_abs_diff_expected << std::endl;
149 std::cout << " obtained: " << summary.max_abs_diff_obtained << std::endl;
150 std::cout << " absolute diff: " << summary.max_abs_diff_value << std::endl;
153 if (summary.max_rel_diff_value > 0)
155 const auto tolerance_level = summary.max_rel_diff_value / FLT_EPSILON;
157 std::cout << " Max relative diff at ["
158 << nnfw::misc::tensor::IndexFormatter(summary.max_rel_diff_index) << "]" << std::endl;
159 std::cout << " expected: " << summary.max_rel_diff_expected << std::endl;
160 std::cout << " obtained: " << summary.max_rel_diff_obtained << std::endl;
161 std::cout << " relative diff: " << summary.max_rel_diff_value << std::endl;
162 std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl;
165 if (diffs.size() > 0)
169 std::cout << " ---- Details ---" << std::endl;
170 for (const auto &diff : diffs)
172 const auto absolute_diff = std::fabs(diff.expected - diff.obtained);
173 const auto relative_diff = nnfw::misc::fp32::relative_diff(diff.expected, diff.obtained);
174 const auto tolerance_level = relative_diff / FLT_EPSILON;
176 std::cout << " Diff at [" << nnfw::misc::tensor::IndexFormatter(diff.index) << "]"
178 std::cout << " expected: " << diff.expected << std::endl;
179 std::cout << " obtained: " << diff.obtained << std::endl;
180 std::cout << " absolute diff: " << absolute_diff << std::endl;
181 std::cout << " relative diff: " << relative_diff << std::endl;
182 std::cout << " (tolerance level = " << tolerance_level << ")" << std::endl;
193 bool TfLiteInterpMatchApp::run(::tflite::Interpreter &interp, ::tflite::Interpreter &nnapi) const
195 assert(interp.outputs() == nnapi.outputs());
197 bool all_matched = true;
199 using Comparator = std::function<bool(int id, ::tflite::Interpreter &, ::tflite::Interpreter &)>;
201 std::map<TfLiteType, Comparator> comparators;
203 comparators[kTfLiteUInt8] = [this](int id, ::tflite::Interpreter &interp,
204 ::tflite::Interpreter &nnapi) {
205 const auto expected = nnfw::tflite::TensorView<uint8_t>::make(interp, id);
206 const auto obtained = nnfw::tflite::TensorView<uint8_t>::make(nnapi, id);
208 return compareSingleTensorView(expected, obtained, id);
211 comparators[kTfLiteInt32] = [this](int id, ::tflite::Interpreter &interp,
212 ::tflite::Interpreter &nnapi) {
213 const auto expected = nnfw::tflite::TensorView<int32_t>::make(interp, id);
214 const auto obtained = nnfw::tflite::TensorView<int32_t>::make(nnapi, id);
216 return compareSingleTensorView(expected, obtained, id);
219 comparators[kTfLiteFloat32] = [this](int id, ::tflite::Interpreter &interp,
220 ::tflite::Interpreter &nnapi) {
221 const auto expected = nnfw::tflite::TensorView<float>::make(interp, id);
222 const auto obtained = nnfw::tflite::TensorView<float>::make(nnapi, id);
224 return compareSingleTensorView(expected, obtained, id);
227 comparators[kTfLiteBool] = [this](int id, ::tflite::Interpreter &interp,
228 ::tflite::Interpreter &nnapi) {
229 const auto expected = nnfw::tflite::TensorView<bool>::make(interp, id);
230 const auto obtained = nnfw::tflite::TensorView<bool>::make(nnapi, id);
232 return compareSingleTensorView(expected, obtained, id);
235 for (const auto &id : interp.outputs())
237 assert(interp.tensor(id)->type == nnapi.tensor(id)->type);
239 auto it = comparators.find(interp.tensor(id)->type);
241 if (it == comparators.end())
243 throw std::runtime_error{"Not supported output type"};
246 const auto &comparator = it->second;
248 if (!comparator(id, interp, nnapi))