2 * Copyright (c) 2020 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "tensorflow/lite/model.h"
18 #include "tensorflow/lite/kernels/register.h"
21 #include "tensor_view.h"
22 #include "misc/EnvVar.h"
23 #include "misc/RandomGenerator.h"
24 #include "misc/tensor/IndexIterator.h"
25 #include "misc/tensor/Object.h"
26 #include "benchmark.h"
34 using namespace std::placeholders; // for _1, _2 ...
36 #define TFLITE_ENSURE(exp) \
38 const TfLiteStatus status = (exp); \
40 if (status != kTfLiteOk) \
42 std::ostringstream ss; \
43 ss << #exp << " failed (" << __FILE__ << ":" << __LINE__ << ")"; \
44 throw std::runtime_error{ss.str()}; \
51 void print_max_idx(float *f, int size)
53 float *p = std::max_element(f, f + size);
54 std::cout << "max:" << p - f;
57 static const char *default_backend_cand = "tflite_cpu";
59 // Verifies whether the model is a flatbuffer file.
60 class BMFlatBufferVerifier : public tflite::TfLiteVerifier
63 bool Verify(const char *data, int length, tflite::ErrorReporter *reporter) override
66 flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(data), length);
67 if (!tflite::VerifyModelBuffer(verifier))
69 reporter->Report("The model is not a valid Flatbuffer file");
76 } // namespace anonymous
78 int main(const int argc, char **argv)
80 tflite::StderrReporter error_reporter;
82 TFLiteVanillaRun::Args args(argc, argv);
84 std::chrono::milliseconds t_model_load(0), t_prepare(0);
86 // TODO Apply verbose level to phases
87 const int verbose = args.getVerboseLevel();
88 benchmark::Phases phases(
89 benchmark::PhaseOption{args.getMemoryPoll(), args.getGpuMemoryPoll(), args.getRunDelay()});
91 std::unique_ptr<tflite::FlatBufferModel> model;
92 std::unique_ptr<tflite::Interpreter> interpreter;
93 std::unique_ptr<tflite::TfLiteVerifier> verifier{new BMFlatBufferVerifier};
97 phases.run("MODEL_LOAD", [&](const benchmark::Phase &, uint32_t) {
98 if (args.getModelValidate())
100 model = tflite::FlatBufferModel::VerifyAndBuildFromFile(args.getTFLiteFilename().c_str(),
101 verifier.get(), &error_reporter);
105 model = tflite::FlatBufferModel::BuildFromFile(args.getTFLiteFilename().c_str(),
108 if (model == nullptr)
110 throw std::runtime_error{"Cannot create model"};
113 // Use tflite's resolver, not onert's one
114 tflite::ops::builtin::BuiltinOpResolver resolver;
115 tflite::InterpreterBuilder builder(*model, resolver);
116 TFLITE_ENSURE(builder(&interpreter))
117 interpreter->SetNumThreads(nnfw::misc::EnvVar("THREAD").asInt(-1));
120 catch (const std::exception &e)
122 std::cerr << e.what() << '\n';
126 const bool use_nnapi = nnfw::misc::EnvVar("USE_NNAPI").asBool(false);
130 phases.run("PREPARE", [&](const benchmark::Phase &, uint32_t) {
131 interpreter->UseNNAPI(use_nnapi);
132 interpreter->AllocateTensors();
135 catch (const std::exception &e)
137 std::cerr << e.what() << '\n';
141 const int seed = 1; /* TODO Add an option for seed value */
142 nnfw::misc::RandomGenerator randgen{seed, 0.0f, 2.0f};
144 // No input specified. So we fill the input tensors with random values.
145 for (const auto &o : interpreter->inputs())
147 TfLiteTensor *tensor = interpreter->tensor(o);
148 if (tensor->type == kTfLiteInt32)
150 // Generate singed 32-bit integer (s32) input
151 auto tensor_view = TFLiteVanillaRun::TensorView<int32_t>::make(*interpreter, o);
155 nnfw::misc::tensor::iterate(tensor_view.shape())
156 << [&](const nnfw::misc::tensor::Index &ind) {
157 // TODO Generate random values
158 // Gather operation: index should be within input coverage.
159 tensor_view.at(ind) = value;
163 else if (tensor->type == kTfLiteUInt8)
165 // Generate unsigned 8-bit integer input
166 auto tensor_view = TFLiteVanillaRun::TensorView<uint8_t>::make(*interpreter, o);
170 nnfw::misc::tensor::iterate(tensor_view.shape())
171 << [&](const nnfw::misc::tensor::Index &ind) {
172 // TODO Generate random values
173 tensor_view.at(ind) = value;
174 value = (value + 1) & 0xFF;
177 else if (tensor->type == kTfLiteBool)
179 // Generate bool input
180 auto tensor_view = TFLiteVanillaRun::TensorView<bool>::make(*interpreter, o);
182 auto fp = static_cast<bool (nnfw::misc::RandomGenerator::*)(
183 const ::nnfw::misc::tensor::Shape &, const ::nnfw::misc::tensor::Index &)>(
184 &nnfw::misc::RandomGenerator::generate<bool>);
185 const nnfw::misc::tensor::Object<bool> data(tensor_view.shape(),
186 std::bind(fp, randgen, _1, _2));
188 nnfw::misc::tensor::iterate(tensor_view.shape())
189 << [&](const nnfw::misc::tensor::Index &ind) {
190 const auto value = data.at(ind);
191 tensor_view.at(ind) = value;
196 assert(tensor->type == kTfLiteFloat32);
198 const float *end = reinterpret_cast<const float *>(tensor->data.raw_const + tensor->bytes);
199 for (float *ptr = tensor->data.f; ptr < end; ptr++)
201 *ptr = randgen.generate<float>();
206 std::cout << "input tensor indices = [";
207 for (const auto &o : interpreter->inputs())
209 std::cout << o << ",";
211 std::cout << "]" << std::endl;
213 // NOTE: Measuring memory can't avoid taking overhead. Therefore, memory will be measured on the
217 phases.run("WARMUP", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); },
218 args.getWarmupRuns());
219 phases.run("EXECUTE", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); },
220 args.getNumRuns(), true);
224 phases.run("WARMUP", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); },
225 [&](const benchmark::Phase &phase, uint32_t nth) {
227 << "warmup " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms"
230 args.getWarmupRuns());
231 phases.run("EXECUTE", [&](const benchmark::Phase &, uint32_t) { interpreter->Invoke(); },
232 [&](const benchmark::Phase &phase, uint32_t nth) {
234 << "run " << nth + 1 << " takes " << phase.time[nth] / 1e3 << " ms"
237 args.getNumRuns(), true);
240 std::cout << "output tensor indices = [";
241 for (const auto &o : interpreter->outputs())
243 std::cout << o << "(";
245 print_max_idx(interpreter->tensor(o)->data.f, interpreter->tensor(o)->bytes / sizeof(float));
249 std::cout << "]" << std::endl;
251 // TODO Apply verbose level to result
254 benchmark::Result result(phases);
257 benchmark::printResult(result);
259 if (args.getWriteReport())
262 std::string exec_basename;
263 std::string model_basename;
264 std::string backend_name = default_backend_cand;
266 std::vector<char> vpath(args.getTFLiteFilename().begin(), args.getTFLiteFilename().end() + 1);
267 model_basename = basename(vpath.data());
268 size_t lastindex = model_basename.find_last_of(".");
269 model_basename = model_basename.substr(0, lastindex);
270 exec_basename = basename(argv[0]);
272 benchmark::writeResult(result, exec_basename, model_basename, backend_name);