2 * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
20 #include <forward_list>
29 #include <boost/filesystem.hpp>
30 #include <boost/format.hpp>
31 #include <boost/program_options.hpp>
37 #include <tensorflow/lite/context.h>
38 #include <tensorflow/lite/interpreter.h>
39 #include <tensorflow/lite/model.h>
42 #include "tflite/ext/nnapi_delegate.h"
43 #include "tflite/ext/kernels/register.h"
45 const std::string kDefaultImagesDir = "res/input/";
46 const std::string kDefaultModelFile = "res/model.tflite";
48 template <typename... Args> void Print(const char *fmt, Args... args)
50 #if __cplusplus >= 201703L
51 std::cerr << boost::str(boost::format(fmt) % ... % std::forward<Args>(args)) << std::endl;
55 unroll{0, (f % std::forward<Args>(args), 0)...};
56 std::cerr << boost::str(f) << std::endl;
60 template <typename DataType> struct BaseLabelData
62 explicit BaseLabelData(int label = -1, DataType confidence = 0)
63 : label(label), confidence(confidence)
67 static std::vector<BaseLabelData<DataType>> FindLabels(const DataType *output_tensor,
68 unsigned int top_n = 5)
70 top_n = top_n > 1000 ? 1000 : top_n;
72 std::vector<size_t> indices(1000);
73 std::generate(indices.begin(), indices.end(), [&n]() { return n++; });
74 std::sort(indices.begin(), indices.end(), [output_tensor](const size_t &i1, const size_t &i2) {
75 return output_tensor[i1] > output_tensor[i2];
77 std::vector<BaseLabelData<DataType>> results(top_n);
78 for (unsigned int i = 0; i < top_n; ++i)
80 results[i].label = indices[i];
81 results[i].confidence = output_tensor[indices[i]];
93 virtual ~BaseRunner() = default;
96 * @brief Run a model for each file in a directory, and collect and print
99 virtual void IterateInDirectory(const std::string &dir_path, const int labels_offset) = 0;
102 * @brief Request that the iteration be stopped after the current file.
104 virtual void ScheduleInterruption() = 0;
107 template <typename DataType_> class Runner : public BaseRunner
110 using DataType = DataType_;
111 using LabelData = BaseLabelData<DataType>;
113 const int kInputSize;
114 const int KOutputSize = 1001 * sizeof(DataType);
116 Runner(std::unique_ptr<tflite::Interpreter> interpreter,
117 std::unique_ptr<tflite::FlatBufferModel> model,
118 std::unique_ptr<::nnfw::tflite::NNAPIDelegate> delegate, unsigned img_size)
119 : interpreter(std::move(interpreter)), model(std::move(model)), delegate(std::move(delegate)),
120 interrupted(false), kInputSize(1 * img_size * img_size * 3 * sizeof(DataType))
122 inference_times.reserve(500);
127 virtual ~Runner() = default;
130 * @brief Get the model's input tensor.
132 virtual DataType *GetInputTensor() = 0;
135 * @brief Get the model's output tensor.
137 virtual DataType *GetOutputTensor() = 0;
140 * @brief Load Image file into tensor.
141 * @return Class number if present in filename, -1 otherwise.
143 virtual int LoadFile(const boost::filesystem::path &input_file)
145 DataType *input_tensor = GetInputTensor();
146 if (input_file.extension() == ".bin")
148 // Load data as raw tensor
149 std::ifstream input_stream(input_file.string(), std::ifstream::binary);
150 input_stream.read(reinterpret_cast<char *>(input_tensor), kInputSize);
151 input_stream.close();
152 int class_num = boost::lexical_cast<int>(input_file.filename().string().substr(0, 4));
157 // Load data as image file
158 throw std::runtime_error("Runner can only load *.bin files");
167 status = delegate->Invoke(interpreter.get());
171 status = interpreter->Invoke();
173 if (status != kTfLiteOk)
175 throw std::runtime_error("Failed to invoke interpreter.");
181 auto t0 = std::chrono::high_resolution_clock::now();
183 auto t1 = std::chrono::high_resolution_clock::now();
184 std::chrono::duration<double> fs = t1 - t0;
185 auto d = std::chrono::duration_cast<std::chrono::milliseconds>(fs);
186 inference_times.push_back(d.count());
187 if (d > std::chrono::milliseconds(10))
189 Print(" -- inference duration: %lld ms", d.count());
193 auto du = std::chrono::duration_cast<std::chrono::microseconds>(fs);
194 Print(" -- inference duration: %lld us", du.count());
199 void DumpOutputTensor(const std::string &output_file)
201 DataType *output_tensor = GetOutputTensor();
202 std::ofstream output_stream(output_file, std::ofstream::binary);
203 output_stream.write(reinterpret_cast<char *>(output_tensor), KOutputSize);
206 void PrintExecutionSummary() const
208 Print("Execution summary:");
209 Print(" -- # of processed images: %d", num_images);
214 // Inference time - mean
215 double mean = std::accumulate(inference_times.begin(), inference_times.end(), 0.0) / num_images;
216 Print(" -- mean inference time: %.1f ms", mean);
217 // Inference time - std
218 std::vector<double> diff(num_images);
219 std::transform(inference_times.begin(), inference_times.end(), diff.begin(),
220 [mean](size_t n) { return n - mean; });
221 double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
222 double std_inference_time = std::sqrt(sq_sum / num_images);
223 Print(" -- std inference time: %.1f ms", std_inference_time);
224 // Top-1 and Top-5 accuracies
225 float num_top1 = std::accumulate(top1.begin(), top1.end(), 0);
226 float num_top5 = std::accumulate(top5.begin(), top5.end(), 0);
227 Print(" -- top1: %.3f, top5: %.3f", num_top1 / num_images, num_top5 / num_images);
230 virtual void ScheduleInterruption() override { interrupted = true; }
232 virtual void IterateInDirectory(const std::string &dir_path, const int labels_offset) override
235 namespace fs = boost::filesystem;
236 if (!fs::is_directory(dir_path))
238 throw std::runtime_error("Could not open input directory.");
241 inference_times.clear();
246 std::vector<LabelData> lds;
247 fs::directory_iterator end;
248 for (auto it = fs::directory_iterator(dir_path); it != end; ++it)
254 if (!fs::is_regular_file(*it))
258 Print("File : %s", it->path().string());
261 class_num = LoadFile(*it) + labels_offset;
262 Print("Class: %d", class_num);
264 catch (std::exception &e)
266 Print("%s", e.what());
269 int status = Process();
272 DataType *output_tensor = GetOutputTensor();
273 lds = LabelData::FindLabels(output_tensor, 5);
274 bool is_top1 = lds[0].label == class_num;
275 bool is_top5 = false;
276 for (const auto &ld : lds)
278 is_top5 = is_top5 || (ld.label == class_num);
279 Print(" -- label: %s (%d), prob: %.3f", ld.label >= 0 ? labels[ld.label] : "", ld.label,
280 static_cast<float>(ld.confidence));
282 Print(" -- top1: %d, top5: %d", is_top1, is_top5);
283 top1.push_back(is_top1);
284 top5.push_back(is_top5);
288 PrintExecutionSummary();
292 std::unique_ptr<tflite::Interpreter> interpreter;
293 std::unique_ptr<tflite::FlatBufferModel> model;
294 std::unique_ptr<::nnfw::tflite::NNAPIDelegate> delegate;
296 std::vector<size_t> inference_times;
297 std::vector<bool> top1;
298 std::vector<bool> top5;
300 std::atomic_bool interrupted;
303 class FloatRunner : public Runner<float>
306 using Runner<float>::DataType;
308 FloatRunner(std::unique_ptr<tflite::Interpreter> interpreter,
309 std::unique_ptr<tflite::FlatBufferModel> model,
310 std::unique_ptr<::nnfw::tflite::NNAPIDelegate> delegate, unsigned img_size)
311 : Runner<float>(std::move(interpreter), std::move(model), std::move(delegate), img_size)
315 virtual ~FloatRunner() = default;
317 virtual DataType *GetInputTensor() override
319 return interpreter->tensor(interpreter->inputs()[0])->data.f;
322 virtual DataType *GetOutputTensor() override
324 return interpreter->tensor(interpreter->outputs()[0])->data.f;
328 class QuantizedRunner : public Runner<uint8_t>
331 using Runner<uint8_t>::DataType;
333 QuantizedRunner(std::unique_ptr<tflite::Interpreter> interpreter,
334 std::unique_ptr<tflite::FlatBufferModel> model,
335 std::unique_ptr<::nnfw::tflite::NNAPIDelegate> delegate, unsigned img_size)
336 : Runner<uint8_t>(std::move(interpreter), std::move(model), std::move(delegate), img_size)
340 virtual ~QuantizedRunner() = default;
342 virtual DataType *GetInputTensor() override
344 return interpreter->tensor(interpreter->inputs()[0])->data.uint8;
347 virtual DataType *GetOutputTensor() override
349 return interpreter->tensor(interpreter->outputs()[0])->data.uint8;
355 TfLiteCpu, /**< Use Tensorflow Lite's CPU kernels. */
356 TfLiteDelegate, /**< Use Tensorflow Lite's NN API delegate. */
357 NnfwDelegate /**< Use NNFW's NN API delegate. */
360 std::unique_ptr<BaseRunner> MakeRunner(const std::string &model_path, unsigned img_size,
361 Target target = Target::NnfwDelegate)
363 auto model = tflite::FlatBufferModel::BuildFromFile(model_path.c_str());
366 throw std::runtime_error(model_path + ": file not found or corrupted.");
368 Print("Model loaded.");
370 std::unique_ptr<tflite::Interpreter> interpreter;
371 nnfw::tflite::BuiltinOpResolver resolver;
372 tflite::InterpreterBuilder(*model, resolver)(&interpreter);
375 throw std::runtime_error("interpreter construction failed.");
377 if (target == Target::TfLiteCpu)
379 interpreter->SetNumThreads(std::max(std::thread::hardware_concurrency(), 1U));
383 interpreter->SetNumThreads(1);
385 if (target == Target::TfLiteDelegate)
387 interpreter->UseNNAPI(true);
390 int input_index = interpreter->inputs()[0];
391 interpreter->ResizeInputTensor(input_index,
392 {1, static_cast<int>(img_size), static_cast<int>(img_size), 3});
393 if (interpreter->AllocateTensors() != kTfLiteOk)
395 throw std::runtime_error("tensor allocation failed.");
398 if (target == Target::TfLiteDelegate)
400 // Do a fake run to load NN API functions.
401 interpreter->Invoke();
404 std::unique_ptr<::nnfw::tflite::NNAPIDelegate> delegate;
405 if (target == Target::NnfwDelegate)
407 delegate.reset(new ::nnfw::tflite::NNAPIDelegate);
408 delegate->BuildGraph(&(interpreter.get()->primary_subgraph()));
411 if (interpreter->tensor(input_index)->type == kTfLiteFloat32)
413 return std::unique_ptr<FloatRunner>(
414 new FloatRunner(std::move(interpreter), std::move(model), std::move(delegate), img_size));
416 else if (interpreter->tensor(input_index)->type == kTfLiteUInt8)
418 return std::unique_ptr<QuantizedRunner>(
419 new QuantizedRunner(std::move(interpreter), std::move(model), std::move(delegate), img_size));
421 throw std::invalid_argument("data type of model's input tensor is not supported.");
424 Target GetTarget(const std::string &str)
426 static const std::map<std::string, Target> target_names{
427 {"tflite-cpu", Target::TfLiteCpu},
428 {"tflite-delegate", Target::TfLiteDelegate},
429 {"nnfw-delegate", Target::NnfwDelegate}};
430 if (target_names.find(str) == target_names.end())
432 throw std::invalid_argument(
433 str + ": invalid target. Run with --help for a list of available targets.");
435 return target_names.at(str);
438 // We need a global pointer to the runner for the SIGINT handler
439 BaseRunner *runner_ptr = nullptr;
440 void HandleSigInt(int)
442 if (runner_ptr != nullptr)
444 Print("Interrupted. Execution will stop after current image.");
445 runner_ptr->ScheduleInterruption();
446 runner_ptr = nullptr;
454 int main(int argc, char *argv[])
457 namespace po = boost::program_options;
458 po::options_description desc("Run a model on multiple binary images and print"
462 ("help", "print this message and quit")
463 ("model", po::value<std::string>()->default_value(kDefaultModelFile), "tflite file")
464 ("input", po::value<std::string>()->default_value(kDefaultImagesDir), "directory with input images")
465 ("offset", po::value<int>()->default_value(1), "labels offset")
466 ("target", po::value<std::string>()->default_value("nnfw-delegate"),
467 "how the model will be run (available targets: tflite-cpu, tflite-delegate, nnfw-delegate)")
468 ("imgsize", po::value<unsigned>()->default_value(224), "the width and height of the image");
470 po::variables_map vm;
471 po::store(po::parse_command_line(argc, argv, desc), vm);
472 if (vm.count("help"))
474 std::cerr << desc << std::endl;
478 auto runner = MakeRunner(vm["model"].as<std::string>(), vm["imgsize"].as<unsigned>(),
479 GetTarget(vm["target"].as<std::string>()));
480 runner_ptr = runner.get();
482 struct sigaction sigint_handler;
483 sigint_handler.sa_handler = HandleSigInt;
484 sigemptyset(&sigint_handler.sa_mask);
485 sigint_handler.sa_flags = 0;
486 sigaction(SIGINT, &sigint_handler, nullptr);
488 Print("Running TensorFlow Lite...");
489 runner->IterateInDirectory(vm["input"].as<std::string>(), vm["offset"].as<int>());
493 catch (std::exception &e)
495 Print("%s: %s", argv[0], e.what());