From 809c504d0a314aaac04b261b252c0418ae3a196c Mon Sep 17 00:00:00 2001 From: Mikhail Treskin Date: Thu, 12 Nov 2020 12:33:23 +0300 Subject: [PATCH] Summary report generator for layer and subgraph tests (#2707) * Initial summary dumper implementation * Handle Tensoriterator body + add parser script * Add support of XML reports merging + report OP names with versions * Remove debug device name change * Fix windows building issue * Add --disable_test_skips command line option * Gtest failure with logging * Change skipping logic and resolve linkage errors caused by extern * Get graph body from Loop * Fix disable_tests_skipping symbol redefinition * Fix inline for currentTestIsDisabled * Rollback get_body for Loop * Handle cases with skip in test SetUp * Report Loop and TI ops along with ops in subgraph body * Resolve some PR comments * Dummy commit to kick pre-commit validation Co-authored-by: Efode, Irina --- .../tests/functional/plugin/cpu/main.cpp | 32 +++ .../tests/functional/plugin/gna/main.cpp | 32 +++ .../tests/functional/plugin/gpu/main.cpp | 32 +++ .../tests/functional/plugin/myriad/main.cpp | 32 +++ .../functional_test_utils/layer_test_utils.cpp | 232 ++++++++++++++++++--- .../functional_test_utils/layer_test_utils.hpp | 88 +++++++- .../layer_tests_summary/summarize.py | 74 +++++++ .../template/report_template.html | 68 ++++++ .../functional_test_utils/skip_tests_config.cpp | 25 +++ .../functional_test_utils/skip_tests_config.hpp | 13 +- 10 files changed, 590 insertions(+), 38 deletions(-) create mode 100644 inference-engine/tests/functional/plugin/cpu/main.cpp create mode 100644 inference-engine/tests/functional/plugin/gna/main.cpp create mode 100644 inference-engine/tests/functional/plugin/gpu/main.cpp create mode 100644 inference-engine/tests/functional/plugin/myriad/main.cpp create mode 100644 inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py create mode 100644 inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html create mode 100644 inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.cpp diff --git a/inference-engine/tests/functional/plugin/cpu/main.cpp b/inference-engine/tests/functional/plugin/cpu/main.cpp new file mode 100644 index 0000000..e47d603 --- /dev/null +++ b/inference-engine/tests/functional/plugin/cpu/main.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "functional_test_utils/layer_test_utils.hpp" + +int main(int argc, char* argv[]) { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false; + bool print_custom_help = false; + for (int i = 0; i < argc; ++i) { + if (std::string(argv[i]) == "--disable_tests_skipping") { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = true; + } + if (std::string(argv[i]) == "--help") { + print_custom_help = true; + } + } + if (print_custom_help) { + std::cout << "Custom command line argument:" << std::endl; + std::cout << " --disable_tests_skipping" << std::endl; + std::cout << " Ignore tests skipping rules and run all the test" << std::endl; + std::cout << " (except those which are skipped with DISABLED prefix)" << std::endl; + std::cout << std::endl; + } + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment); + auto retcode = RUN_ALL_TESTS(); + + return retcode; +} diff --git a/inference-engine/tests/functional/plugin/gna/main.cpp b/inference-engine/tests/functional/plugin/gna/main.cpp new file mode 100644 index 0000000..6b0bcf6 --- /dev/null +++ b/inference-engine/tests/functional/plugin/gna/main.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "functional_test_utils/layer_test_utils.hpp" + +int main(int argc, char* argv[]) { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false; + bool print_custom_help = false; + for (int i = 0; i < argc; ++i) { + if (std::string(argv[i]) == "--disable_tests_skipping") { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = true; + } + if (std::string(argv[i]) == "--help") { + print_custom_help = true; + } + } + if (print_custom_help) { + std::cout << "Custom command line argument:" << std::endl; + std::cout << " --disable_tests_skipping" << std::endl; + std::cout << " Ignore tests skipping rules and run all the test" << std::endl; + std::cout << " (except those which are skipped with DISABLED prefix)" << std::endl; + std::cout << std::endl; + } + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment); + auto retcode = RUN_ALL_TESTS(); + + return retcode; +} \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/gpu/main.cpp b/inference-engine/tests/functional/plugin/gpu/main.cpp new file mode 100644 index 0000000..6b0bcf6 --- /dev/null +++ b/inference-engine/tests/functional/plugin/gpu/main.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "functional_test_utils/layer_test_utils.hpp" + +int main(int argc, char* argv[]) { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false; + bool print_custom_help = false; + for (int i = 0; i < argc; ++i) { + if (std::string(argv[i]) == "--disable_tests_skipping") { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = true; + } + if (std::string(argv[i]) == "--help") { + print_custom_help = true; + } + } + if (print_custom_help) { + std::cout << "Custom command line argument:" << std::endl; + std::cout << " --disable_tests_skipping" << std::endl; + std::cout << " Ignore tests skipping rules and run all the test" << std::endl; + std::cout << " (except those which are skipped with DISABLED prefix)" << std::endl; + std::cout << std::endl; + } + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment); + auto retcode = RUN_ALL_TESTS(); + + return retcode; +} \ No newline at end of file diff --git a/inference-engine/tests/functional/plugin/myriad/main.cpp b/inference-engine/tests/functional/plugin/myriad/main.cpp new file mode 100644 index 0000000..6b0bcf6 --- /dev/null +++ b/inference-engine/tests/functional/plugin/myriad/main.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "functional_test_utils/layer_test_utils.hpp" + +int main(int argc, char* argv[]) { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = false; + bool print_custom_help = false; + for (int i = 0; i < argc; ++i) { + if (std::string(argv[i]) == "--disable_tests_skipping") { + FuncTestUtils::SkipTestsConfig::disable_tests_skipping = true; + } + if (std::string(argv[i]) == "--help") { + print_custom_help = true; + } + } + if (print_custom_help) { + std::cout << "Custom command line argument:" << std::endl; + std::cout << " --disable_tests_skipping" << std::endl; + std::cout << " Ignore tests skipping rules and run all the test" << std::endl; + std::cout << " (except those which are skipped with DISABLED prefix)" << std::endl; + std::cout << std::endl; + } + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new LayerTestsUtils::TestEnvironment); + auto retcode = RUN_ALL_TESTS(); + + return retcode; +} \ No newline at end of file diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.cpp b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.cpp index 78ba9d2..6ea47fd 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.cpp +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.cpp @@ -1,25 +1,191 @@ // Copyright (C) 2019-2020 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // +#include #include #include - +#include +#include #include "layer_test_utils.hpp" #include "plugin_config.hpp" namespace LayerTestsUtils { +Summary *Summary::p_instance = nullptr; +SummaryDestroyer Summary::destroyer; + +SummaryDestroyer::~SummaryDestroyer() { + delete p_instance; +} + +void SummaryDestroyer::initialize(Summary *p) { + p_instance = p; +} + +Summary &Summary::getInstance() { + if (!p_instance) { + p_instance = new Summary(); + destroyer.initialize(p_instance); + } + return *p_instance; +} + +void Summary::updateOPsStats(ngraph::NodeTypeInfo op, PassRate::Statuses status) { + auto it = opsStats.find(op); + if (it != opsStats.end()) { + auto &passrate = it->second; + switch (status) { + case PassRate::PASSED: + passrate.passed += 1; + break; + case PassRate::FAILED: + passrate.failed += 1; + break; + case PassRate::SKIPPED: + passrate.skipped += 1; + break; + } + } else { + switch (status) { + case PassRate::PASSED: + opsStats[op] = PassRate(1, 0, 0); + break; + case PassRate::FAILED: + opsStats[op] = PassRate(0, 1, 0); + break; + case PassRate::SKIPPED: + opsStats[op] = PassRate(0, 0, 1); + break; + } + } +} + +void TestEnvironment::TearDown() { + std::vector opsets; + opsets.push_back(ngraph::get_opset1()); + opsets.push_back(ngraph::get_opset2()); + opsets.push_back(ngraph::get_opset3()); + opsets.push_back(ngraph::get_opset4()); + opsets.push_back(ngraph::get_opset5()); + std::set opsInfo; + for (const auto &opset : opsets) { + const auto &type_info_set = opset.get_type_info_set(); + opsInfo.insert(type_info_set.begin(), type_info_set.end()); + } + + auto &s = Summary::getInstance(); + auto stats = s.getOPsStats(); + + pugi::xml_document doc; + + std::ifstream file; + file.open(reportFileName); + + time_t rawtime; + struct tm *timeinfo; + char timeNow[80]; + + time(&rawtime); + // cpplint require to use localtime_r instead which is not available in C++14 + timeinfo = localtime(&rawtime); // NOLINT + + strftime(timeNow, sizeof(timeNow), "%d-%m-%Y %H:%M:%S", timeinfo); + + pugi::xml_node root; + if (file) { + doc.load_file(reportFileName.c_str()); + root = doc.child("report"); + //Ugly but shorter than to write predicate for find_atrribute() to update existing one + root.remove_attribute("timestamp"); + root.append_attribute("timestamp").set_value(timeNow); + + root.remove_child("ops_list"); + root.child("results").remove_child(s.deviceName.c_str()); + } else { + root = doc.append_child("report"); + root.append_attribute("timestamp").set_value(timeNow); + root.append_child("results"); + } + + pugi::xml_node opsNode = root.append_child("ops_list"); + for (const auto &op : opsInfo) { + std::string name = std::string(op.name) + "-" + std::to_string(op.version); + pugi::xml_node entry = opsNode.append_child(name.c_str()); + } + + pugi::xml_node resultsNode = root.child("results"); + pugi::xml_node currentDeviceNode = resultsNode.append_child(s.deviceName.c_str()); + for (const auto &it : stats) { + std::string name = std::string(it.first.name) + "-" + std::to_string(it.first.version); + pugi::xml_node entry = currentDeviceNode.append_child(name.c_str()); + entry.append_attribute("passed").set_value(std::to_string(it.second.passed).c_str()); + entry.append_attribute("failed").set_value(std::to_string(it.second.failed).c_str()); + entry.append_attribute("skipped").set_value(std::to_string(it.second.skipped).c_str()); + entry.append_attribute("passrate").set_value(std::to_string(it.second.getPassrate()).c_str()); + } + bool result = doc.save_file(reportFileName.c_str()); + if (!result) { + std::cout << "Failed to write report to " << reportFileName << "!" << std::endl; + } +} + LayerTestsCommon::LayerTestsCommon() : threshold(1e-2f) { core = PluginCache::get().ie(targetDevice); } void LayerTestsCommon::Run() { - SKIP_IF_CURRENT_TEST_IS_DISABLED() + auto &s = Summary::getInstance(); + s.setDeviceName(targetDevice); + auto reportStatus = [this, &s](PassRate::Statuses status) { + if (function){ + for (const auto &op : function->get_ordered_ops()) { + if (ngraph::is_type(op) || + ngraph::is_type(op) || + ngraph::is_type(op)) { + continue; + } else if (ngraph::is_type(op)) { + s.updateOPsStats(op->get_type_info(), status); + auto ti = ngraph::as_type_ptr(op); + auto ti_body = ti->get_function(); + for (const auto &ti_op : ti_body->get_ordered_ops()) { + s.updateOPsStats(ti_op->get_type_info(), status); + } + } else if (ngraph::is_type(op)) { + s.updateOPsStats(op->get_type_info(), status); + auto loop = ngraph::as_type_ptr(op); + auto loop_body = loop->get_function(); + for (const auto &loop_op : loop_body->get_ordered_ops()) { + s.updateOPsStats(loop_op->get_type_info(), status); + } + } else { + s.updateOPsStats(op->get_type_info(), status); + } + } + } + }; - LoadNetwork(); - Infer(); - Validate(); + if (FuncTestUtils::SkipTestsConfig::currentTestIsDisabled()) { + reportStatus(PassRate::Statuses::SKIPPED); + GTEST_SKIP() << "Disabled test due to configuration" << std::endl; + } + + try { + LoadNetwork(); + Infer(); + Validate(); + reportStatus(PassRate::Statuses::PASSED); + } + catch (const std::runtime_error &re) { + reportStatus(PassRate::Statuses::FAILED); + GTEST_FATAL_FAILURE_(re.what()); + } catch (const std::exception &ex) { + reportStatus(PassRate::Statuses::FAILED); + GTEST_FATAL_FAILURE_(ex.what()); + } catch (...) { + reportStatus(PassRate::Statuses::FAILED); + GTEST_FATAL_FAILURE_("Unknown failure occurred."); + } } InferenceEngine::Blob::Ptr LayerTestsCommon::GenerateInput(const InferenceEngine::InputInfo &info) const { @@ -39,29 +205,37 @@ void LayerTestsCommon::Compare(const std::vector &expected, const const auto &size = actual->size(); switch (precision) { case InferenceEngine::Precision::FP32: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, threshold); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, threshold); break; case InferenceEngine::Precision::I32: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; case InferenceEngine::Precision::I64: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; case InferenceEngine::Precision::I8: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; case InferenceEngine::Precision::U16: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; case InferenceEngine::Precision::I16: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; case InferenceEngine::Precision::BOOL: case InferenceEngine::Precision::U8: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; case InferenceEngine::Precision::U64: - Compare(reinterpret_cast(expectedBuffer), reinterpret_cast(actualBuffer), size, 0); + Compare(reinterpret_cast(expectedBuffer), + reinterpret_cast(actualBuffer), size, 0); break; default: FAIL() << "Comparator for " << precision << " precision isn't supported"; @@ -69,7 +243,7 @@ void LayerTestsCommon::Compare(const std::vector &expected, const } void LayerTestsCommon::Compare(const InferenceEngine::Blob::Ptr &expected, const InferenceEngine::Blob::Ptr &actual) { - auto get_raw_buffer = [] (const InferenceEngine::Blob::Ptr &blob) { + auto get_raw_buffer = [](const InferenceEngine::Blob::Ptr &blob) { auto memory = InferenceEngine::as(blob); IE_ASSERT(memory); const auto lockedMemory = memory->wmap(); @@ -142,29 +316,32 @@ void LayerTestsCommon::Infer() { std::vector> LayerTestsCommon::CalculateRefs() { // nGraph interpreter does not support f16 // IE converts f16 to f32 - ngraph::pass::ConvertPrecision().run_on_function(function); + ngraph::pass::ConvertPrecision().run_on_function( + function); function->validate_nodes_and_infer_types(); auto referenceInputs = std::vector>(inputs.size()); for (std::size_t i = 0; i < inputs.size(); ++i) { - const auto& input = inputs[i]; - const auto& inputSize = input->byteSize(); + const auto &input = inputs[i]; + const auto &inputSize = input->byteSize(); - auto& referenceInput = referenceInputs[i]; + auto &referenceInput = referenceInputs[i]; referenceInput.resize(inputSize); auto memory = InferenceEngine::as(input); IE_ASSERT(memory); const auto lockedMemory = memory->wmap(); - const auto buffer = lockedMemory.as(); + const auto buffer = lockedMemory.as(); std::copy(buffer, buffer + inputSize, referenceInput.data()); } auto ieOutPrc = outPrc; const auto &actualOutputs = GetOutputs(); - std::vector convertType(actualOutputs.size(), FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(ieOutPrc)); + std::vector convertType(actualOutputs.size(), + FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(ieOutPrc)); if (ieOutPrc == InferenceEngine::Precision::UNSPECIFIED) { for (size_t i = 0; i < convertType.size(); i++) { - convertType[i] = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(actualOutputs[i]->getTensorDesc().getPrecision()); + convertType[i] = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc( + actualOutputs[i]->getTensorDesc().getPrecision()); } } @@ -208,24 +385,25 @@ std::vector LayerTestsCommon::GetOutputs() { return outputs; } -void LayerTestsCommon::Compare(const std::vector>& expectedOutputs, const std::vector& actualOutputs) { +void LayerTestsCommon::Compare(const std::vector> &expectedOutputs, + const std::vector &actualOutputs) { for (std::size_t outputIndex = 0; outputIndex < expectedOutputs.size(); ++outputIndex) { - const auto& expected = expectedOutputs[outputIndex]; - const auto& actual = actualOutputs[outputIndex]; + const auto &expected = expectedOutputs[outputIndex]; + const auto &actual = actualOutputs[outputIndex]; Compare(expected, actual); } } void LayerTestsCommon::Validate() { auto expectedOutputs = CalculateRefs(); - const auto& actualOutputs = GetOutputs(); + const auto &actualOutputs = GetOutputs(); if (expectedOutputs.empty()) { return; } IE_ASSERT(actualOutputs.size() == expectedOutputs.size()) - << "nGraph interpreter has " << expectedOutputs.size() << " outputs, while IE " << actualOutputs.size(); + << "nGraph interpreter has " << expectedOutputs.size() << " outputs, while IE " << actualOutputs.size(); Compare(expectedOutputs, actualOutputs); } @@ -238,7 +416,7 @@ std::shared_ptr LayerTestsCommon::GetFunction() { return function; } -std::map& LayerTestsCommon::GetConfiguration() { +std::map &LayerTestsCommon::GetConfiguration() { return configuration; } } // namespace LayerTestsUtils diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.hpp b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.hpp index 7d4e6b5..319e84e 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.hpp +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_test_utils.hpp @@ -27,9 +27,94 @@ #include "ngraph_functions/utils/ngraph_helpers.hpp" #include "ngraph_functions/pass/convert_prc.hpp" - namespace LayerTestsUtils { +class Summary; + +class SummaryDestroyer { +private: + Summary *p_instance; +public: + ~SummaryDestroyer(); + + void initialize(Summary *p); +}; + +class TestEnvironment; + +class LayerTestsCommon; + +struct PassRate { + enum Statuses { + PASSED, + FAILED, + SKIPPED + }; + unsigned long passed = 0; + unsigned long failed = 0; + unsigned long skipped = 0; + + PassRate() = default; + + PassRate(unsigned long p, unsigned long f, unsigned long s) { + passed = p; + failed = f; + skipped = s; + } + + float getPassrate() const { + if (passed == 0 && failed == 0) { + return 0.; + } else if (passed != 0 && failed == 0) { + return 100.; + } else { + return (passed / (passed + failed)) * 100.; + } + } +}; + +class Summary { +private: + static Summary *p_instance; + static SummaryDestroyer destroyer; + std::map opsStats = {}; + std::string deviceName; + +protected: + Summary() = default; + + Summary(const Summary &); + + Summary &operator=(Summary &); + + ~Summary() = default; + + void updateOPsStats(ngraph::NodeTypeInfo op, PassRate::Statuses status); + + std::map getOPsStats() { return opsStats; } + + std::string getDeviceName() const { return deviceName; } + + void setDeviceName(std::string device) { deviceName = device; } + + friend class SummaryDestroyer; + + friend class TestEnvironment; + + friend class LayerTestsCommon; + +public: + static Summary &getInstance(); +}; + +class TestEnvironment : public ::testing::Environment { +public: + void TearDown() override; + +private: + std::string reportFileName = "report.xml"; +}; + using TargetDevice = std::string; typedef std::tuple< @@ -128,6 +213,7 @@ protected: virtual void Validate(); virtual std::vector> CalculateRefs(); + std::vector GetOutputs(); InferenceEngine::InferRequest inferRequest; diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py new file mode 100644 index 0000000..7ddf2b8 --- /dev/null +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/summarize.py @@ -0,0 +1,74 @@ +import xml.etree.ElementTree as ET +from jinja2 import Environment, FileSystemLoader +import argparse +import os +from datetime import datetime + +parser = argparse.ArgumentParser() + +xml_help = """ +Paths to xml summary files from layer tests. +In case of entries intersection, results will +be merged basing on timestamp - entry from latest +report is be kept. +""" +out_help = "Path where to save html report" + +parser.add_argument("--xml", help=xml_help, nargs="*", required=True) +parser.add_argument("--out", help=out_help, default="") +args = parser.parse_args() + + +def merge_xmls(xmls): + if len(xmls) == 1: + return xmls[0] + summary = ET.Element("report") + summary.set("timestamp", xmls[0].attrib["timestamp"]) + results = ET.SubElement(summary, "results") + ops_list = ET.SubElement(summary, "ops_list") + for xml in xmls: + for op in xml.find("ops_list"): + if ops_list.find(op.tag) is None: + ET.SubElement(ops_list, op.tag) + for device in xml.find("results"): + device_results = results.find(device.tag) + if device_results is None: + results.append(device) + else: + for entry in device: + if device_results.find(entry.tag) is not None: + current_timestamp = datetime.strptime(xml.attrib["timestamp"], "%d-%m-%Y %H:%M:%S") + base_timestamp = datetime.strptime(summary.attrib["timestamp"], "%d-%m-%Y %H:%M:%S") + if current_timestamp > base_timestamp: + device_results.find(entry.tag).attrib = entry.attrib + else: + device_results.append(entry) + return summary + + +xmls = [] +for xml in args.xml: + xmls.append(ET.parse(xml).getroot()) + +root = merge_xmls(xmls) +timestamp = root.attrib["timestamp"] +ops = [] +for op in root.find("ops_list"): + ops.append(op.tag) +ordered_ops = sorted(ops) +results = {} +for device in root.find("results"): + results[device.tag] = {op.tag: op.attrib for op in device} + for op in results[device.tag]: + results[device.tag][op]["passrate"] = round(float(results[device.tag][op]["passrate"]), 1) + +devices = results.keys() + +file_loader = FileSystemLoader('template') +env = Environment(loader=file_loader) +template = env.get_template('report_template.html') + +res = template.render(ordered_ops=ordered_ops, devices=devices, results=results, timestamp=timestamp) + +with open(os.path.join(args.out, "report.html"), "w") as f: + f.write(res) diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html new file mode 100644 index 0000000..e95f74d --- /dev/null +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/layer_tests_summary/template/report_template.html @@ -0,0 +1,68 @@ + + + + + + + + + + + Report + + + + + + +
+

Operations coverage summary {{ timestamp }}

+
+ + + + + {% for d in devices -%} + + {% endfor %} + + + + {% for op in ordered_ops -%} + + + {% for d in devices -%} + {% if op in results[d] -%} + + {% else -%} + + {% endif -%} + + {% endfor %} + + {% endfor -%} + + + {% for d in devices -%} + + {% endfor %} + + + +
Operation{{ d }}
{{ op }} + {{ results[d][op].passrate }}% (p:{{ results[d][op].passed }}, + f:{{ results[d][op].failed }},s:{{ results[d][op].skipped }}) + No tests
Total: {{ordered_ops|length}}{{results[d]|length}}
+ + \ No newline at end of file diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.cpp b/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.cpp new file mode 100644 index 0000000..d92a02b --- /dev/null +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.cpp @@ -0,0 +1,25 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "functional_test_utils/skip_tests_config.hpp" + +namespace FuncTestUtils { +namespace SkipTestsConfig { + +bool disable_tests_skipping = false; + +bool currentTestIsDisabled() { + bool skip_test = false; + const auto fullName = ::testing::UnitTest::GetInstance()->current_test_info()->test_case_name() + + std::string(".") + ::testing::UnitTest::GetInstance()->current_test_info()->name(); + for (const auto &pattern : disabledTestPatterns()) { + std::regex re(pattern); + if (std::regex_match(fullName, re)) + skip_test = true; + } + return skip_test && !disable_tests_skipping; +} + +} // namespace SkipTestsConfig +} // namespace FuncTestUtils diff --git a/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.hpp b/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.hpp index a119693..fd6a4ba 100644 --- a/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.hpp +++ b/inference-engine/tests/ie_test_utils/functional_test_utils/skip_tests_config.hpp @@ -14,16 +14,9 @@ std::vector disabledTestPatterns(); namespace FuncTestUtils { namespace SkipTestsConfig { -inline bool currentTestIsDisabled() { - const auto fullName = ::testing::UnitTest::GetInstance()->current_test_info()->test_case_name() - + std::string(".") + ::testing::UnitTest::GetInstance()->current_test_info()->name(); - for (const auto &pattern : disabledTestPatterns()) { - std::regex re(pattern); - if (std::regex_match(fullName, re)) - return true; - } - return false; -} +extern bool disable_tests_skipping; + +bool currentTestIsDisabled(); } // namespace SkipTestsConfig } // namespace FuncTestUtils -- 2.7.4