2 * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "tflite/RandomTestRunner.h"
18 #include "tflite/Diff.h"
19 #include "tflite/TensorLogger.h"
20 #include "tflite/ext/nnapi_delegate.h"
22 #include <misc/tensor/IndexIterator.h>
23 #include <misc/tensor/Object.h>
24 #include <misc/EnvVar.h>
25 #include <misc/fp32.h>
37 using namespace std::placeholders;
39 void RandomTestRunner::compile(const nnfw::tflite::Builder &builder)
41 _tfl_interp = builder.build();
42 _nnapi = builder.build();
44 _tfl_interp->UseNNAPI(false);
47 _tfl_interp->AllocateTensors();
48 _nnapi->AllocateTensors();
50 assert(_tfl_interp->inputs() == _nnapi->inputs());
52 using ::tflite::Interpreter;
53 using Initializer = std::function<void(int id, Interpreter *, Interpreter *)>;
55 std::map<TfLiteType, Initializer> initializers;
56 std::map<TfLiteType, Initializer> reseters;
58 // Generate singed 32-bit integer (s32) input
59 initializers[kTfLiteInt32] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
60 assert(_tfl_interp->tensor(id)->type == kTfLiteInt32);
61 assert(_nnapi->tensor(id)->type == kTfLiteInt32);
63 auto tfl_interp_view = nnfw::tflite::TensorView<int32_t>::make(*tfl_interp, id);
64 auto nnapi_view = nnfw::tflite::TensorView<int32_t>::make(*nnapi, id);
66 assert(tfl_interp_view.shape() == nnapi_view.shape());
70 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
71 << [&](const nnfw::misc::tensor::Index &ind) {
72 // TODO Generate random values
73 tfl_interp_view.at(ind) = value;
74 nnapi_view.at(ind) = value;
79 // Generate singed 32-bit integer (s32) input
80 reseters[kTfLiteInt32] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
81 assert(_tfl_interp->tensor(id)->type == kTfLiteInt32);
82 assert(_nnapi->tensor(id)->type == kTfLiteInt32);
84 auto tfl_interp_view = nnfw::tflite::TensorView<int32_t>::make(*tfl_interp, id);
85 auto nnapi_view = nnfw::tflite::TensorView<int32_t>::make(*nnapi, id);
87 assert(tfl_interp_view.shape() == nnapi_view.shape());
91 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
92 << [&](const nnfw::misc::tensor::Index &ind) {
93 // TODO Generate random values
94 tfl_interp_view.at(ind) = value;
95 nnapi_view.at(ind) = value;
99 initializers[kTfLiteUInt8] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
100 assert(_tfl_interp->tensor(id)->type == kTfLiteUInt8);
101 assert(_nnapi->tensor(id)->type == kTfLiteUInt8);
103 auto tfl_interp_view = nnfw::tflite::TensorView<uint8_t>::make(*tfl_interp, id);
104 auto nnapi_view = nnfw::tflite::TensorView<uint8_t>::make(*nnapi, id);
106 assert(tfl_interp_view.shape() == nnapi_view.shape());
108 auto fp = static_cast<uint8_t (nnfw::misc::RandomGenerator::*)(
109 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
110 &nnfw::misc::RandomGenerator::generate<uint8_t>);
111 const nnfw::misc::tensor::Object<uint8_t> data(tfl_interp_view.shape(),
112 std::bind(fp, _randgen, _1, _2));
113 assert(tfl_interp_view.shape() == data.shape());
115 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
116 << [&](const nnfw::misc::tensor::Index &ind) {
117 const auto value = data.at(ind);
119 tfl_interp_view.at(ind) = value;
120 nnapi_view.at(ind) = value;
124 reseters[kTfLiteUInt8] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
125 assert(_tfl_interp->tensor(id)->type == kTfLiteUInt8);
126 assert(_nnapi->tensor(id)->type == kTfLiteUInt8);
128 auto tfl_interp_view = nnfw::tflite::TensorView<uint8_t>::make(*tfl_interp, id);
129 auto nnapi_view = nnfw::tflite::TensorView<uint8_t>::make(*nnapi, id);
131 assert(tfl_interp_view.shape() == nnapi_view.shape());
133 auto fp = static_cast<uint8_t (nnfw::misc::RandomGenerator::*)(
134 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
135 &nnfw::misc::RandomGenerator::generate<uint8_t>);
136 const nnfw::misc::tensor::Object<uint8_t> data(tfl_interp_view.shape(),
137 std::bind(fp, _randgen, _1, _2));
138 assert(tfl_interp_view.shape() == data.shape());
142 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
143 << [&](const nnfw::misc::tensor::Index &ind) {
144 tfl_interp_view.at(ind) = value;
145 nnapi_view.at(ind) = value;
149 initializers[kTfLiteFloat32] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
150 assert(_tfl_interp->tensor(id)->type == kTfLiteFloat32);
151 assert(_nnapi->tensor(id)->type == kTfLiteFloat32);
153 auto tfl_interp_view = nnfw::tflite::TensorView<float>::make(*tfl_interp, id);
154 auto nnapi_view = nnfw::tflite::TensorView<float>::make(*nnapi, id);
156 assert(tfl_interp_view.shape() == nnapi_view.shape());
158 auto fp = static_cast<float (nnfw::misc::RandomGenerator::*)(
159 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
160 &nnfw::misc::RandomGenerator::generate<float>);
161 const nnfw::misc::tensor::Object<float> data(tfl_interp_view.shape(),
162 std::bind(fp, _randgen, _1, _2));
164 assert(tfl_interp_view.shape() == data.shape());
166 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
167 << [&](const nnfw::misc::tensor::Index &ind) {
168 const auto value = data.at(ind);
170 tfl_interp_view.at(ind) = value;
171 nnapi_view.at(ind) = value;
175 reseters[kTfLiteFloat32] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
176 assert(_tfl_interp->tensor(id)->type == kTfLiteFloat32);
177 assert(_nnapi->tensor(id)->type == kTfLiteFloat32);
179 auto tfl_interp_view = nnfw::tflite::TensorView<float>::make(*tfl_interp, id);
180 auto nnapi_view = nnfw::tflite::TensorView<float>::make(*nnapi, id);
182 assert(tfl_interp_view.shape() == nnapi_view.shape());
184 auto fp = static_cast<float (nnfw::misc::RandomGenerator::*)(
185 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
186 &nnfw::misc::RandomGenerator::generate<float>);
187 const nnfw::misc::tensor::Object<float> data(tfl_interp_view.shape(),
188 std::bind(fp, _randgen, _1, _2));
190 assert(tfl_interp_view.shape() == data.shape());
194 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
195 << [&](const nnfw::misc::tensor::Index &ind) {
196 tfl_interp_view.at(ind) = value;
197 nnapi_view.at(ind) = value;
201 initializers[kTfLiteBool] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
202 assert(_tfl_interp->tensor(id)->type == kTfLiteBool);
203 assert(_nnapi->tensor(id)->type == kTfLiteBool);
205 auto tfl_interp_view = nnfw::tflite::TensorView<bool>::make(*tfl_interp, id);
206 auto nnapi_view = nnfw::tflite::TensorView<bool>::make(*nnapi, id);
208 assert(tfl_interp_view.shape() == nnapi_view.shape());
210 auto fp = static_cast<bool (nnfw::misc::RandomGenerator::*)(
211 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
212 &nnfw::misc::RandomGenerator::generate<bool>);
213 const nnfw::misc::tensor::Object<bool> data(tfl_interp_view.shape(),
214 std::bind(fp, _randgen, _1, _2));
216 assert(tfl_interp_view.shape() == data.shape());
218 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
219 << [&](const nnfw::misc::tensor::Index &ind) {
220 const auto value = data.at(ind);
222 tfl_interp_view.at(ind) = value;
223 nnapi_view.at(ind) = value;
227 reseters[kTfLiteBool] = [&](int id, Interpreter *tfl_interp, Interpreter *nnapi) {
228 assert(_tfl_interp->tensor(id)->type == kTfLiteBool);
229 assert(_nnapi->tensor(id)->type == kTfLiteBool);
231 auto tfl_interp_view = nnfw::tflite::TensorView<bool>::make(*tfl_interp, id);
232 auto nnapi_view = nnfw::tflite::TensorView<bool>::make(*nnapi, id);
234 assert(tfl_interp_view.shape() == nnapi_view.shape());
236 auto fp = static_cast<bool (nnfw::misc::RandomGenerator::*)(
237 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
238 &nnfw::misc::RandomGenerator::generate<bool>);
239 const nnfw::misc::tensor::Object<bool> data(tfl_interp_view.shape(),
240 std::bind(fp, _randgen, _1, _2));
242 assert(tfl_interp_view.shape() == data.shape());
246 nnfw::misc::tensor::iterate(tfl_interp_view.shape())
247 << [&](const nnfw::misc::tensor::Index &ind) {
248 tfl_interp_view.at(ind) = value;
249 nnapi_view.at(ind) = value;
253 // Fill IFM with random numbers
254 for (const auto id : _tfl_interp->inputs())
256 assert(_tfl_interp->tensor(id)->type == _nnapi->tensor(id)->type);
258 auto it = initializers.find(_tfl_interp->tensor(id)->type);
260 if (it == initializers.end())
262 throw std::runtime_error{"Not supported input type"};
265 it->second(id, _tfl_interp.get(), _nnapi.get());
269 for (const auto id : _tfl_interp->outputs())
271 assert(_tfl_interp->tensor(id)->type == _nnapi->tensor(id)->type);
273 auto it = reseters.find(_tfl_interp->tensor(id)->type);
275 if (it == reseters.end())
277 throw std::runtime_error{"Not supported input type"};
280 it->second(id, _tfl_interp.get(), _nnapi.get());
284 int RandomTestRunner::run(size_t running_count)
286 std::cout << "[NNAPI TEST] Run T/F Lite Interpreter without NNAPI" << std::endl;
287 _tfl_interp->Invoke();
289 nnfw::tflite::NNAPIDelegate d;
291 for (size_t i = 1; i <= running_count; ++i)
293 std::cout << "[NNAPI TEST #" << i << "] Run T/F Lite Interpreter with NNAPI" << std::endl;
295 char *env = getenv("UPSTREAM_DELEGATE");
297 if (env && !std::string(env).compare("1"))
299 _nnapi->UseNNAPI(true);
305 // primary_subgraph: Experimental interface. Return 1st sugbraph
306 // Invoke() will call BuildGraph() internally
307 if (d.Invoke(&_nnapi.get()->primary_subgraph()))
309 throw std::runtime_error{"Failed to BuildGraph"};
314 std::cout << "[NNAPI TEST #" << i << "] Compare the result" << std::endl;
316 const auto tolerance = _param.tolerance;
318 auto equals = [tolerance](float lhs, float rhs) {
319 // NOTE Hybrid approach
320 // TODO Allow users to set tolerance for absolute_epsilon_equal
321 if (nnfw::misc::fp32::absolute_epsilon_equal(lhs, rhs))
326 return nnfw::misc::fp32::epsilon_equal(lhs, rhs, tolerance);
329 nnfw::misc::tensor::Comparator comparator(equals);
330 TfLiteInterpMatchApp app(comparator);
332 app.verbose() = _param.verbose;
334 bool res = app.run(*_tfl_interp, *_nnapi);
341 std::cout << "[NNAPI TEST #" << i << "] PASSED" << std::endl << std::endl;
343 if (_param.tensor_logging)
344 nnfw::tflite::TensorLogger::get().save(_param.log_path, *_tfl_interp);
350 RandomTestRunner RandomTestRunner::make(uint32_t seed)
352 RandomTestParam param;
354 param.verbose = nnfw::misc::EnvVar("VERBOSE").asInt(0);
355 param.tolerance = nnfw::misc::EnvVar("TOLERANCE").asInt(1);
356 param.tensor_logging = nnfw::misc::EnvVar("TENSOR_LOGGING").asBool(false);
357 param.log_path = nnfw::misc::EnvVar("TENSOR_LOGGING").asString("tensor_log.txt");
359 return RandomTestRunner{seed, param};
362 } // namespace tflite