find_package(InferenceEngineDeveloperPackage REQUIRED)
-add_subdirectory(common)
+add_subdirectory(time-testhelper)
+add_subdirectory(src)
--- /dev/null
+# Time Tests
+
+This test suite contains pipelines, which are executables. The pipelines measure
+the time of their execution, both total and partial. A Python runner calls the
+pipelines and calcuates the average execution time.
+
+## Prerequisites
+
+To build the time tests, you need to have the `build` folder, which is created
+when you configure and build OpenVINO™.
+
+## Measure Time
+
+To build and run the tests, open a terminal and run the commands below:
+
+1. Build tests:
+``` bash
+cmake .. -DInferenceEngineDeveloperPackage_DIR=../../../build && make time-tests
+```
+
+2. Run test:
+``` bash
+./run_executable.py ../../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
+```
+
+++ /dev/null
-# Copyright (C) 2018-2019 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-#
-
-set (TARGET_NAME "TimeTests")
-
-file (GLOB SRC
- *.cpp
- ../ftti_pipeline/*.cpp)
-
-file (GLOB HDR
- *.h
- ../ftti_pipeline/*.h)
-
-# Create library file from sources.
-add_executable(${TARGET_NAME} ${HDR} ${SRC})
-
-find_package(gflags REQUIRED)
-
-target_link_libraries(${TARGET_NAME}
- gflags
- ${InferenceEngine_LIBRARIES}
- )
+++ /dev/null
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include <string>
-#include <vector>
-#include <gflags/gflags.h>
-#include <iostream>
-
-/// @brief message for help argument
-static const char help_message[] = "Print a usage message";
-
-/// @brief message for model argument
-static const char model_message[] = "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to a .blob files with a trained compiled model.";
-
-/// @brief message for target device argument
-static const char target_device_message[] = "Required. Specify a target device to infer on. " \
-"Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO plugin. " \
-"Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI plugin. " \
-"The application looks for a suitable plugin for the specified device.";
-
-/// @brief message for statistics path argument
-static const char statistics_path_message[] = "Required. Path to a file to write statistics.";
-
-/// @brief Define flag for showing help message <br>
-DEFINE_bool(h, false, help_message);
-
-/// @brief Declare flag for showing help message <br>
-DECLARE_bool(help);
-
-/// @brief Define parameter for set model file <br>
-/// It is a required parameter
-DEFINE_string(m, "", model_message);
-
-/// @brief Define parameter for set target device to infer on <br>
-/// It is a required parameter
-DEFINE_string(d, "", target_device_message);
-
-/// @brief Define parameter for set path to a file to write statistics <br>
-/// It is a required parameter
-DEFINE_string(s, "", statistics_path_message);
-
-/**
-* @brief This function show a help message
-*/
-static void showUsage() {
- std::cout << std::endl;
- std::cout << "TimeTests [OPTION]" << std::endl;
- std::cout << "Options:" << std::endl;
- std::cout << std::endl;
- std::cout << " -h, --help " << help_message << std::endl;
- std::cout << " -m \"<path>\" " << model_message << std::endl;
- std::cout << " -d \"<device>\" " << target_device_message << std::endl;
- std::cout << " -s \"<path>\" " << statistics_path_message << std::endl;
-}
+++ /dev/null
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#include "cli.h"
-#include "statistics_writer.h"
-#include "../ftti_pipeline/ftti_pipeline.h"
-
-#include <iostream>
-
-/**
-* @brief Parses command line and check required arguments
-*/
-bool parseAndCheckCommandLine(int argc, char **argv) {
- gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
- if (FLAGS_help || FLAGS_h) {
- showUsage();
- return false;
- }
-
- if (FLAGS_m.empty())
- throw std::logic_error("Model is required but not set. Please set -m option.");
-
- if (FLAGS_d.empty())
- throw std::logic_error("Device is required but not set. Please set -d option.");
-
- if (FLAGS_s.empty())
- throw std::logic_error("Statistics file path is required but not set. Please set -s option.");
-
- return true;
-}
-
-
-/**
-* @brief Function calls `runPipeline` with mandatory time tracking of full run
-*/
-int _runPipeline() {
- SCOPED_TIMER(full_run);
- return runPipeline(FLAGS_m, FLAGS_d);
-}
-
-
-/**
-* @brief Main entry point
-*/
-int main(int argc, char **argv) {
- if (!parseAndCheckCommandLine(argc, argv))
- return -1;
-
- StatisticsWriter::Instance().setFile(FLAGS_s);
- return _runPipeline();
-}
\ No newline at end of file
+++ /dev/null
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include <string>
-#include <sstream>
-#include <fstream>
-#include <cstdio>
-
-
-/**
- * @brief Class response for writing provided statistics
- *
- * Object of the class is writing provided statistics to a specified
- * file in YAML format.
- */
-class StatisticsWriter {
-private:
- std::ofstream statistics_file;
-
- StatisticsWriter() = default;
- StatisticsWriter(const StatisticsWriter&) = delete;
- StatisticsWriter& operator=(const StatisticsWriter&) = delete;
-public:
- /**
- * @brief Creates StatisticsWriter singleton object
- */
- static StatisticsWriter& Instance(){
- static StatisticsWriter writer;
- return writer;
- }
-
- /**
- * @brief Specifies, opens and validates statistics path for writing
- */
- void setFile(const std::string &statistics_path) {
- statistics_file.open(statistics_path);
- if (!statistics_file.good()) {
- std::stringstream err;
- err << "Statistic file \"" << statistics_path << "\" can't be used for writing";
- throw std::runtime_error(err.str());
- }
- }
-
- /**
- * @brief Writes provided statistics in YAML format.
- */
- void write(const std::pair<std::string, float> &record) {
- if (!statistics_file)
- throw std::runtime_error("Statistic file path isn't set");
- statistics_file << record.first << ": " << record.second << "\n";
- }
-};
+++ /dev/null
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include <string>
-#include <chrono>
-#include <fstream>
-#include <memory>
-
-#include "statistics_writer.h"
-
-using time_point = std::chrono::high_resolution_clock::time_point;
-
-/**
-* @brief Class response for encapsulating time measurements.
-*
-* Object of a class measures time at start and finish of object's life cycle.
-* When deleting, reports duration.
-*/
-class Timer {
-private:
- std::string name;
- time_point start_time;
-
-public:
- /**
- * @brief Constructs Timer object and measures start time
- */
- Timer(const std::string &timer_name) {
- name = timer_name;
- start_time = std::chrono::high_resolution_clock::now();
- }
-
- /**
- * @brief Destructs Timer object, measures duration and reports it
- */
- ~Timer(){
- float duration = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::high_resolution_clock::now() - start_time).count();
- StatisticsWriter::Instance().write({name, duration});
- }
-};
-
-#define SCOPED_TIMER(timer_name) Timer timer_name(#timer_name);
+++ /dev/null
-// Copyright (C) 2020 Intel Corporation
-// SPDX-License-Identifier: Apache-2.0
-//
-
-#pragma once
-
-#include "../common/timer.h"
-
-#include <inference_engine.hpp>
-using namespace InferenceEngine;
-
-
-/**
-* @brief Function that contain executable pipeline which will be called from main().
-* The function should not throw any exceptions and responsible for handling it by itself.
-*/
-int runPipeline(const std::string &model, const std::string &device) {
- auto pipeline = [](const std::string &model, const std::string &device){
- SCOPED_TIMER(first_time_to_inference);
-
- Core ie;
- CNNNetwork cnnNetwork;
- ExecutableNetwork exeNetwork;
-
- {
- SCOPED_TIMER(read_network);
- cnnNetwork = ie.ReadNetwork(model);
- }
-
- {
- SCOPED_TIMER(load_network);
- ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
- }
- };
-
- try {
- pipeline(model, device);
- } catch (const InferenceEngine::details::InferenceEngineException& iex) {
- std::cerr << "Inference Engine pipeline failed with Inference Engine exception:\n" << iex.what();
- return 1;
- } catch (const std::exception& ex) {
- std::cerr << "Inference Engine pipeline failed with exception:\n" << ex.what();
- return 2;
- } catch (...) {
- std::cerr << "Inference Engine pipeline failed\n";
- return 3;
- }
- return 0;
-}
\ No newline at end of file
--- /dev/null
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# add dummy `time_tests` target combines all time tests
+add_custom_target(time_tests)
+
+# Build test from every source file matchs *-pipeline.cpp.
+# Test target name is source file name without extension.
+FILE(GLOB tests "*-pipeline.cpp")
+
+foreach(test_source ${tests})
+ get_filename_component(test_name ${test_source} NAME_WE)
+ add_executable(${test_name} ${test_source})
+
+ target_link_libraries(${test_name} PRIVATE IE::inference_engine time-testhelper)
+
+ add_dependencies(time_tests ${test_name})
+endforeach()
--- /dev/null
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <inference_engine.hpp>
+#include <iostream>
+
+#include "time-testhelper/timer.h"
+using namespace InferenceEngine;
+
+/**
+ * @brief Function that contain executable pipeline which will be called from
+ * main(). The function should not throw any exceptions and responsible for
+ * handling it by itself.
+ */
+int runPipeline(const std::string &model, const std::string &device) {
+ auto pipeline = [](const std::string &model, const std::string &device) {
+ SCOPED_TIMER(first_time_to_inference);
+
+ Core ie;
+ CNNNetwork cnnNetwork;
+ ExecutableNetwork exeNetwork;
+
+ {
+ SCOPED_TIMER(read_network);
+ cnnNetwork = ie.ReadNetwork(model);
+ }
+
+ {
+ SCOPED_TIMER(load_network);
+ ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
+ }
+ };
+
+ try {
+ pipeline(model, device);
+ } catch (const InferenceEngine::details::InferenceEngineException &iex) {
+ std::cerr
+ << "Inference Engine pipeline failed with Inference Engine exception:\n"
+ << iex.what();
+ return 1;
+ } catch (const std::exception &ex) {
+ std::cerr << "Inference Engine pipeline failed with exception:\n"
+ << ex.what();
+ return 2;
+ } catch (...) {
+ std::cerr << "Inference Engine pipeline failed\n";
+ return 3;
+ }
+ return 0;
+}
\ No newline at end of file
--- /dev/null
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set (TARGET_NAME "time-testhelper")
+
+find_package(gflags REQUIRED)
+
+file (GLOB SRC *.cpp)
+add_library(${TARGET_NAME} STATIC ${SRC})
+target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
+
+target_link_libraries(${TARGET_NAME} gflags)
--- /dev/null
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <gflags/gflags.h>
+#include <iostream>
+#include <string>
+#include <vector>
+
+/// @brief message for help argument
+static const char help_message[] = "Print a usage message";
+
+/// @brief message for model argument
+static const char model_message[] =
+ "Required. Path to an .xml/.onnx/.prototxt file with a trained model or to "
+ "a .blob files with a trained compiled model.";
+
+/// @brief message for target device argument
+static const char target_device_message[] =
+ "Required. Specify a target device to infer on. "
+ "Use \"-d HETERO:<comma-separated_devices_list>\" format to specify HETERO "
+ "plugin. "
+ "Use \"-d MULTI:<comma-separated_devices_list>\" format to specify MULTI "
+ "plugin. "
+ "The application looks for a suitable plugin for the specified device.";
+
+/// @brief message for statistics path argument
+static const char statistics_path_message[] =
+ "Required. Path to a file to write statistics.";
+
+/// @brief Define flag for showing help message <br>
+DEFINE_bool(h, false, help_message);
+
+/// @brief Declare flag for showing help message <br>
+DECLARE_bool(help);
+
+/// @brief Define parameter for set model file <br>
+/// It is a required parameter
+DEFINE_string(m, "", model_message);
+
+/// @brief Define parameter for set target device to infer on <br>
+/// It is a required parameter
+DEFINE_string(d, "", target_device_message);
+
+/// @brief Define parameter for set path to a file to write statistics <br>
+/// It is a required parameter
+DEFINE_string(s, "", statistics_path_message);
+
+/**
+ * @brief This function show a help message
+ */
+static void showUsage() {
+ std::cout << std::endl;
+ std::cout << "TimeTests [OPTION]" << std::endl;
+ std::cout << "Options:" << std::endl;
+ std::cout << std::endl;
+ std::cout << " -h, --help " << help_message << std::endl;
+ std::cout << " -m \"<path>\" " << model_message << std::endl;
+ std::cout << " -d \"<device>\" " << target_device_message
+ << std::endl;
+ std::cout << " -s \"<path>\" " << statistics_path_message
+ << std::endl;
+}
--- /dev/null
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <chrono>
+#include <string>
+
+namespace TimeTest {
+using time_point = std::chrono::high_resolution_clock::time_point;
+
+/** Encapsulate time measurements.
+Object of a class measures time at start and finish of object's life cycle.
+When destroyed, reports duration.
+*/
+class Timer {
+private:
+ std::string name;
+ time_point start_time;
+
+public:
+ /// Constructs Timer object and measures start time.
+ Timer(const std::string &timer_name);
+
+ /// Destructs Timer object, measures duration and reports it.
+ ~Timer();
+};
+
+#define SCOPED_TIMER(timer_name) TimeTest::Timer timer_name(#timer_name);
+
+} // namespace TimeTest
\ No newline at end of file
--- /dev/null
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "cli.h"
+#include "statistics_writer.h"
+#include "time-testhelper/timer.h"
+
+#include <iostream>
+
+int runPipeline(const std::string &model, const std::string &device);
+
+/**
+ * @brief Parses command line and check required arguments
+ */
+bool parseAndCheckCommandLine(int argc, char **argv) {
+ gflags::ParseCommandLineNonHelpFlags(&argc, &argv, true);
+ if (FLAGS_help || FLAGS_h) {
+ showUsage();
+ return false;
+ }
+
+ if (FLAGS_m.empty())
+ throw std::logic_error(
+ "Model is required but not set. Please set -m option.");
+
+ if (FLAGS_d.empty())
+ throw std::logic_error(
+ "Device is required but not set. Please set -d option.");
+
+ if (FLAGS_s.empty())
+ throw std::logic_error(
+ "Statistics file path is required but not set. Please set -s option.");
+
+ return true;
+}
+
+/**
+ * @brief Function calls `runPipeline` with mandatory time tracking of full run
+ */
+int _runPipeline() {
+ SCOPED_TIMER(full_run);
+ return runPipeline(FLAGS_m, FLAGS_d);
+}
+
+/**
+ * @brief Main entry point
+ */
+int main(int argc, char **argv) {
+ if (!parseAndCheckCommandLine(argc, argv))
+ return -1;
+
+ StatisticsWriter::Instance().setFile(FLAGS_s);
+ return _runPipeline();
+}
\ No newline at end of file
--- /dev/null
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <cstdio>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+/**
+ * @brief Class response for writing provided statistics
+ *
+ * Object of the class is writing provided statistics to a specified
+ * file in YAML format.
+ */
+class StatisticsWriter {
+private:
+ std::ofstream statistics_file;
+
+ StatisticsWriter() = default;
+ StatisticsWriter(const StatisticsWriter &) = delete;
+ StatisticsWriter &operator=(const StatisticsWriter &) = delete;
+
+public:
+ /**
+ * @brief Creates StatisticsWriter singleton object
+ */
+ static StatisticsWriter &Instance() {
+ static StatisticsWriter writer;
+ return writer;
+ }
+
+ /**
+ * @brief Specifies, opens and validates statistics path for writing
+ */
+ void setFile(const std::string &statistics_path) {
+ statistics_file.open(statistics_path);
+ if (!statistics_file.good()) {
+ std::stringstream err;
+ err << "Statistic file \"" << statistics_path
+ << "\" can't be used for writing";
+ throw std::runtime_error(err.str());
+ }
+ }
+
+ /**
+ * @brief Writes provided statistics in YAML format.
+ */
+ void write(const std::pair<std::string, float> &record) {
+ if (!statistics_file)
+ throw std::runtime_error("Statistic file path isn't set");
+ statistics_file << record.first << ": " << record.second << "\n";
+ }
+};
--- /dev/null
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "time-testhelper/timer.h"
+#include <chrono>
+#include <fstream>
+#include <memory>
+#include <string>
+
+#include "statistics_writer.h"
+
+using time_point = std::chrono::high_resolution_clock::time_point;
+
+namespace TimeTest {
+
+Timer::Timer(const std::string &timer_name) {
+ name = timer_name;
+ start_time = std::chrono::high_resolution_clock::now();
+}
+
+Timer::~Timer() {
+ float duration = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::high_resolution_clock::now() - start_time)
+ .count();
+ StatisticsWriter::Instance().write({name, duration});
+}
+
+} // namespace TimeTest
\ No newline at end of file
projects / platform / upstream / dldt.git / commitdiff