* [Stress] Redesigned MemCheckTests: 1. Added MemCheckPipeline to incapsulate measures and logging. 2. Moved references to array
* [Stress] Added tracking of THREADS in MemCheckTests
#include <gtest/gtest.h>
-#define checkRefVmValues() \
- if (!Environment::Instance().getCollectResultsOnly()) { \
- ASSERT_GT(test_refs.ref_vmsize, 0) << "Reference value of VmSize is less than 0. Value: " \
- << test_refs.ref_vmsize; \
- ASSERT_GT(test_refs.ref_vmsize, 0) << "Reference value of VmPeak is less than 0. Value: " \
- << test_refs.ref_vmpeak; \
- ASSERT_GT(test_refs.ref_vmrss, 0) << "Reference value of VmRSS is less than 0. Value: " \
- << test_refs.ref_vmrss; \
- ASSERT_GT(test_refs.ref_vmrss, 0) << "Reference value of VmHWM is less than 0. Value: " \
- << test_refs.ref_vmhwm; \
- }
+#include <inference_engine.hpp>
+
+using namespace InferenceEngine;
+
class MemCheckTestSuite : public ::testing::TestWithParam<TestCase> {
+public:
+ std::string test_name, model, model_name, device;
+ TestReferences test_refs;
+
+ void SetUp() override {
+ const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
+ test_name = std::string(test_info->name()).substr(0, std::string(test_info->name()).find('/'));
+ //const std::string full_test_name = std::string(test_info->test_case_name()) + "." + std::string(test_info->name());
+
+ const auto& test_params = GetParam();
+ model = test_params.model;
+ model_name = test_params.model_name;
+ device = test_params.device;
+
+ test_refs.collect_vm_values_for_test(test_name, test_params);
+ if (!Environment::Instance().getCollectResultsOnly()) {
+ ASSERT_GT(test_refs.references[VMSIZE], 0) << "Reference value of VmSize is less than 0. Value: "
+ << test_refs.references[VMSIZE];
+ ASSERT_GT(test_refs.references[VMPEAK], 0) << "Reference value of VmPeak is less than 0. Value: "
+ << test_refs.references[VMPEAK];
+ ASSERT_GT(test_refs.references[VMRSS], 0) << "Reference value of VmRSS is less than 0. Value: "
+ << test_refs.references[VMRSS];
+ ASSERT_GT(test_refs.references[VMHWM], 0) << "Reference value of VmHWM is less than 0. Value: "
+ << test_refs.references[VMHWM];
+ }
+ }
};
// tests_pipelines/tests_pipelines.cpp
TEST_P(MemCheckTestSuite, create_exenetwork) {
- std::string test_name = "create_exenetwork";
- auto test_params = GetParam();
+ log_info("Create ExecutableNetwork from network: \"" << model
+ << "\" for device: \"" << device << "\"");
+ auto test_pipeline = [&]{
+ MemCheckPipeline memCheckPipeline;
- TestReferences test_refs;
- test_refs.collect_vm_values_for_test(test_name, test_params);
+ Core ie;
+ ie.GetVersions(device);
+ CNNNetwork cnnNetwork = ie.ReadNetwork(model);
+ ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
- checkRefVmValues();
+ log_info("Memory consumption after LoadNetwork:");
+ memCheckPipeline.record_measures(test_name);
- TestResult res = test_create_exenetwork(test_params.model_name, test_params.model, test_params.device,
- test_refs.ref_vmsize, test_refs.ref_vmpeak, test_refs.ref_vmrss,
- test_refs.ref_vmhwm);
+ log_debug(memCheckPipeline.get_reference_record_for_test(test_name, model_name, device));
+ return memCheckPipeline.measure();
+ };
+
+ TestResult res = common_test_pipeline(test_pipeline, test_refs.references);
EXPECT_EQ(res.first, TestStatus::TEST_OK) << res.second;
}
TEST_P(MemCheckTestSuite, infer_request_inference) {
- std::string test_name = "infer_request_inference";
- auto test_params = GetParam();
+ log_info("Inference of InferRequest from network: \"" << model
+ << "\" for device: \"" << device << "\"");
+ auto test_pipeline = [&]{
+ MemCheckPipeline memCheckPipeline;
- TestReferences test_refs;
- test_refs.collect_vm_values_for_test(test_name, test_params);
+ Core ie;
+ ie.GetVersions(device);
+ CNNNetwork cnnNetwork = ie.ReadNetwork(model);
+ ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, device);
+ InferRequest inferRequest = exeNetwork.CreateInferRequest();
+ inferRequest.Infer();
+ OutputsDataMap output_info(cnnNetwork.getOutputsInfo());
+ for (auto &output : output_info)
+ Blob::Ptr outputBlob = inferRequest.GetBlob(output.first);
+
+ log_info("Memory consumption after Inference:");
+ memCheckPipeline.record_measures(test_name);
- checkRefVmValues();
+ log_debug(memCheckPipeline.get_reference_record_for_test(test_name, model_name, device));
+ return memCheckPipeline.measure();
+ };
- TestResult res = test_infer_request_inference(test_params.model_name, test_params.model, test_params.device,
- test_refs.ref_vmsize, test_refs.ref_vmpeak, test_refs.ref_vmrss,
- test_refs.ref_vmhwm);
+ TestResult res = common_test_pipeline(test_pipeline, test_refs.references);
EXPECT_EQ(res.first, TestStatus::TEST_OK) << res.second;
}
// tests_pipelines/tests_pipelines.cpp
#include <string>
#include <math.h>
-#include <chrono>
-
-#include <inference_engine.hpp>
#define REPORTING_THRESHOLD 1.3
+// delimiter used for measurements print. Should be compatible with script parses tests logs
+#define MEMCHECK_DELIMITER "\t\t"
-using namespace InferenceEngine;
-
-#define getAlignedVmValues(vmsize, vmpeak, vmrss, vmhwm, vmsize_to_align, vmrss_to_align) \
- getVmValues(test_cur_vmsize, test_cur_vmpeak, test_cur_vmrss, test_cur_vmhwm); \
- test_cur_vmsize -= vmsize_before_test; \
- test_cur_vmpeak -= vmsize_before_test; \
- test_cur_vmrss -= vmrss_before_test; \
- test_cur_vmhwm -= vmrss_before_test;
-
-#define log_debug_ref_record_for_test(test_name) \
- log_debug("Record to update reference config: " \
- << "<model path=\"" + model_name + "\"" + " test=\"" + test_name + "\" device=\"" + \
- target_device + \
- "\" vmsize=\"" + std::to_string((int) (test_cur_vmsize * REPORTING_THRESHOLD)) + \
- "\" vmpeak=\"" + std::to_string((int) (test_cur_vmpeak * REPORTING_THRESHOLD)) + \
- "\" vmrss=\"" + std::to_string((int) (test_cur_vmrss * REPORTING_THRESHOLD)) + \
- "\" vmhwm=\"" + std::to_string((int) (test_cur_vmhwm * REPORTING_THRESHOLD)) + "\" />");
-
-#define log_info_ref_mem_usage() \
- log_info("Reference values of virtual memory consumption:"); \
- log_info("VMRSS\t\tVMHWM\t\tVMSIZE\t\tVMPEAK"); \
- log_info(ref_vmrss << "\t\t" << ref_vmhwm << "\t\t" << ref_vmsize << "\t\t" << ref_vmpeak);
-
-#define log_info_cur_mem_usage() \
- log_info("Current values of virtual memory consumption:"); \
- log_info("VMRSS\t\tVMHWM\t\tVMSIZE\t\tVMPEAK"); \
- log_info(test_cur_vmrss << "\t\t" << test_cur_vmhwm << "\t\t" << test_cur_vmsize << "\t\t" << test_cur_vmpeak);
-
-TestResult
-test_create_exenetwork(const std::string &model_name, const std::string &model_path, const std::string &target_device,
- const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss, const long &ref_vmhwm) {
- log_info("Create ExecutableNetwork from network: \"" << model_path
- << "\" for device: \"" << target_device << "\"");
- long vmsize_before_test = 0, vmrss_before_test = 0,
- test_cur_vmsize = 0, test_cur_vmpeak = 0,
- test_cur_vmrss = 0, test_cur_vmhwm = 0;
- vmsize_before_test = (long) getVmSizeInKB();
- vmrss_before_test = (long) getVmRSSInKB();
-
- Core ie;
- CNNNetwork cnnNetwork = ie.ReadNetwork(model_path);
- ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, target_device);
-
- getAlignedVmValues(test_cur_vmsize, test_cur_vmpeak, test_cur_vmrss, test_cur_vmhwm,
- vmsize_before_test, vmrss_before_test);
-
- log_debug_ref_record_for_test("create_exenetwork");
- log_info_ref_mem_usage();
- log_info_cur_mem_usage();
-
- if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmrss > ref_vmrss))
- return TestResult(TestStatus::TEST_FAILED,
- "Test failed: RSS virtual memory consumption became greater than reference.\n"
- "Reference RSS memory consumption: " + std::to_string(ref_vmrss) + " KB.\n" +
- "Current RSS memory consumption: " + std::to_string(test_cur_vmrss) + " KB.\n");
-
- if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmhwm > ref_vmhwm))
- return TestResult(TestStatus::TEST_FAILED,
- "Test failed: HWM (peak of RSS) virtual memory consumption is greater than reference.\n"
- "Reference HWM of memory consumption: " + std::to_string(ref_vmhwm) + " KB.\n" +
- "Current HWM of memory consumption: " + std::to_string(test_cur_vmhwm) + " KB.\n");
-
- return TestResult(TestStatus::TEST_OK, "");
+MemCheckPipeline::MemCheckPipeline() {
+ start_measures[VMRSS] = (long) getVmRSSInKB();
+ start_measures[VMHWM] = start_measures[VMRSS];
+ start_measures[VMSIZE] = (long) getVmSizeInKB();
+ start_measures[VMPEAK] = start_measures[VMSIZE];
+ start_measures[THREADS] = (long) getThreadsNum();
}
-TestResult
-test_infer_request_inference(const std::string &model_name, const std::string &model_path,
- const std::string &target_device,
- const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss,
- const long &ref_vmhwm) {
- log_info("Inference of InferRequest from network: \"" << model_path
- << "\" for device: \"" << target_device << "\"");
- long vmsize_before_test = 0, vmrss_before_test = 0,
- test_cur_vmsize = 0, test_cur_vmpeak = 0,
- test_cur_vmrss = 0, test_cur_vmhwm = 0;
- std::chrono::system_clock::time_point t_start, t_end;
- std::chrono::duration<double> t_diff;
-
- vmsize_before_test = (long) getVmSizeInKB();
- vmrss_before_test = (long) getVmRSSInKB();
-
- Core ie;
- CNNNetwork cnnNetwork = ie.ReadNetwork(model_path);
- ExecutableNetwork exeNetwork = ie.LoadNetwork(cnnNetwork, target_device);
- InferRequest infer_request = exeNetwork.CreateInferRequest();
+std::array<long, MeasureValueMax> MemCheckPipeline::_measure() {
+ std::array<long, MeasureValueMax> measures;
+ measures[VMRSS] = (long) getVmRSSInKB();
+ measures[VMHWM] = (long) getVmHWMInKB();
+ measures[VMSIZE] = (long) getVmSizeInKB();
+ measures[VMPEAK] = (long) getVmPeakInKB();
+ measures[THREADS] = (long) getThreadsNum(); // TODO: resolve *-32295
+ return measures;
+}
- log_info_ref_mem_usage();
+std::array<long, MeasureValueMax> MemCheckPipeline::measure() {
+ std::array<long, MeasureValueMax> measures = _measure();
+ std::transform(std::begin(measures), std::end(measures), std::begin(start_measures), std::begin(measures),
+ [](long measure, long start_measure) -> long {
+ return measure - start_measure;
+ });
+ return measures;
+}
- t_start = std::chrono::system_clock::now();
- int seconds = 1;
- do {
- infer_request.Infer();
- OutputsDataMap output_info(cnnNetwork.getOutputsInfo());
- for (auto &output : output_info)
- Blob::Ptr outputBlob = infer_request.GetBlob(output.first);
- t_end = std::chrono::system_clock::now();
- t_diff = t_end - t_start;
+void MemCheckPipeline::record_measures(const std::string & id) {
+ std::array<long, MeasureValueMax> measures = measure();
+ log_debug("[ MEASURE ] " << MEMCHECK_DELIMITER << id);
+ log_info(util::get_measure_values_headers(MEMCHECK_DELIMITER));
+ log_info(util::get_measure_values_as_str(measures, MEMCHECK_DELIMITER));
+}
- getAlignedVmValues(test_cur_vmsize, test_cur_vmpeak, test_cur_vmrss, test_cur_vmhwm,
- vmsize_before_test, vmrss_before_test);
+std::string MemCheckPipeline::get_reference_record_for_test(std::string test_name, std::string model_name,
+ std::string target_device) {
+ std::array<long, MeasureValueMax> measures = measure();
+ std::stringstream ss;
+ ss << "Record to update reference config: "
+ << "<model path=\"" << model_name << "\"" <<
+ " test=\"" << test_name << "\" device=\"" << target_device <<
+ "\" vmsize=\"" << (int) (measures[VMSIZE] * REPORTING_THRESHOLD) <<
+ "\" vmpeak=\"" << (int) (measures[VMPEAK] * REPORTING_THRESHOLD) <<
+ "\" vmrss=\"" << (int) (measures[VMRSS] * REPORTING_THRESHOLD) <<
+ "\" vmhwm=\"" << (int) (measures[VMHWM] * REPORTING_THRESHOLD) << "\" />";
+ return ss.str();
+}
- if (t_diff.count() > (double) (seconds)) {
- log_info("Current values of virtual memory consumption after " << seconds << " seconds:");
- log_info("VMRSS\t\tVMHWM\t\tVMSIZE\t\tVMPEAK");
- log_info(test_cur_vmrss << "\t\t" << test_cur_vmhwm << "\t\t" << test_cur_vmsize << "\t\t" << test_cur_vmpeak);
- seconds++;
- }
- } while (t_diff.count() < 5);
+TestResult common_test_pipeline(const std::function<std::array<long, MeasureValueMax>()>& test_pipeline,
+ const std::array<long, MeasureValueMax> &references) {
+ log_info("Reference values of virtual memory consumption:");
+ log_info(util::get_measure_values_headers(MEMCHECK_DELIMITER));
+ log_info(util::get_measure_values_as_str(references, MEMCHECK_DELIMITER));
- log_debug_ref_record_for_test("infer_request_inference");
+ std::array<long, MeasureValueMax> measures = test_pipeline();
- if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmrss > ref_vmrss))
+ if ((!Environment::Instance().getCollectResultsOnly()) && (measures[VMRSS] > references[VMRSS]))
return TestResult(TestStatus::TEST_FAILED,
"Test failed: RSS virtual memory consumption became greater than reference.\n"
- "Reference RSS memory consumption: " + std::to_string(ref_vmrss) + " KB.\n" +
- "Current RSS memory consumption: " + std::to_string(test_cur_vmrss) + " KB.\n");
+ "Reference RSS memory consumption: " + std::to_string(references[VMRSS]) + " KB.\n" +
+ "Current RSS memory consumption: " + std::to_string(measures[VMRSS]) + " KB.\n");
- if ((!Environment::Instance().getCollectResultsOnly()) && (test_cur_vmhwm > ref_vmhwm))
+ if ((!Environment::Instance().getCollectResultsOnly()) && (measures[VMHWM] > references[VMHWM]))
return TestResult(TestStatus::TEST_FAILED,
"Test failed: HWM (peak of RSS) virtual memory consumption is greater than reference.\n"
- "Reference HWM of memory consumption: " + std::to_string(ref_vmhwm) + " KB.\n" +
- "Current HWM of memory consumption: " + std::to_string(test_cur_vmhwm) + " KB.\n");
+ "Reference HWM of memory consumption: " + std::to_string(references[VMHWM]) + " KB.\n" +
+ "Current HWM of memory consumption: " + std::to_string(measures[VMHWM]) + " KB.\n");
return TestResult(TestStatus::TEST_OK, "");
}
#pragma once
+#include "../tests_utils.h"
#include "../../common/tests_utils.h"
#include "../../common/utils.h"
#include <string>
// tests_pipelines/tests_pipelines.cpp
-TestResult test_create_exenetwork(const std::string &model_name, const std::string &model_path, const std::string &target_device,
- const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss, const long &ref_vmhwm);
-TestResult test_infer_request_inference(const std::string &model_name, const std::string &model_path, const std::string &target_device,
- const long &ref_vmsize, const long &ref_vmpeak, const long &ref_vmrss, const long &ref_vmhwm);
+/**
+ * @brief Class response for encapsulating measure and measurements printing
+ *
+ * Current class measures only in scope of it's lifetime. In this case need
+ * to note that deletion of objects created before class creation may lead
+ * to negative values because of alignment on starting values.
+ * Also deletion of objects created in scope of class lifetime may decrease
+ * values computed on previous measure.
+ */
+class MemCheckPipeline {
+private:
+ std::array<long, MeasureValueMax> start_measures; // measures before run (will be used as baseline)
+
+ /**
+ * @brief Measures values at the current point of time
+ */
+ std::array<long, MeasureValueMax> _measure();
+public:
+ /**
+ * @brief Constructs MemCheckPipeline object and
+ * measure values to use as baseline
+ */
+ MemCheckPipeline();
+
+ /**
+ * @brief Measures values at the current point of time and
+ * returns measurements aligned on a baseline
+ */
+ std::array<long, MeasureValueMax> measure();
+
+ /**
+ * @brief Measures values and records aligned measurements using provided identifier
+ * provided identifier
+ */
+ void record_measures(const std::string & id);
+
+ /**
+ * @brief Prepares string used for fast generation of file with references
+ */
+ std::string get_reference_record_for_test(std::string test_name, std::string model_name,
+ std::string target_device);
+};
+
+TestResult common_test_pipeline(const std::function<std::array<long, MeasureValueMax>()>& test_pipeline,
+ const std::array<long, MeasureValueMax> &references);
// tests_pipelines/tests_pipelines.cpp
// SPDX-License-Identifier: Apache-2.0
//
+#pragma once
+
#include "../common/tests_utils.h"
#include <pugixml.hpp>
+// Measure values
+enum MeasureValue { VMRSS = 0, VMHWM, VMSIZE, VMPEAK, THREADS, MeasureValueMax };
+// Measure values headers
+const std::array<std::string, MeasureValueMax> MeasureValueHeader { "VMRSS", "VMHWM", "VMSIZE", "VMPEAK", "THREADS" };
+
+namespace util {
+ template <typename Type>
+ static std::string get_measure_values_as_str(const std::array<Type, MeasureValueMax> & array,
+ const std::string & delimiter = "\t\t") {
+ std::string str = std::to_string(*array.begin());
+ for (auto it = array.begin() + 1; it != array.end(); it++)
+ str += delimiter + std::to_string(*it);
+ return str;
+ }
+
+ static std::string get_measure_values_headers(const std::string & delimiter = "\t\t") {
+ std::string str = *MeasureValueHeader.begin();
+ for (auto it = MeasureValueHeader.begin() + 1; it != MeasureValueHeader.end(); it++)
+ str += delimiter + *it;
+ return str;
+ }
+}
+
class MemCheckEnvironment {
private:
pugi::xml_document _refs_config;
std::vector<std::string> model_path_v, test_name_v, device_v;
std::vector<long> vmsize_v, vmpeak_v, vmrss_v, vmhwm_v;
public:
- long ref_vmsize = -1, ref_vmpeak = -1, ref_vmrss = -1, ref_vmhwm = -1;
+ std::array<long, MeasureValueMax> references;
TestReferences () {
+ std::fill(references.begin(), references.end(), -1);
+
// Parse RefsConfig from MemCheckEnvironment
std::string models_path = Environment::Instance().getEnvConfig()
.child("attributes").child("irs_path").child("value").text().as_string();
if (test_name_v[i] == test_name)
if (model_path_v[i] == test_params.model)
if (device_v[i] == test_params.device) {
- ref_vmsize = vmsize_v[i];
- ref_vmpeak = vmpeak_v[i];
- ref_vmrss = vmrss_v[i];
- ref_vmhwm = vmhwm_v[i];
+ references[VMSIZE] = vmsize_v[i];
+ references[VMPEAK] = vmpeak_v[i];
+ references[VMRSS] = vmrss_v[i];
+ references[VMHWM] = vmhwm_v[i];
}
}
};
PRODUCT_NAME = 'dldt' # product name from build manifest
RE_GTEST_MODEL_XML = re.compile(r'<model[^>]*>')
-RE_GTEST_CUR_MEASURE = re.compile(
- r'Current values of virtual memory consumption')
+RE_GTEST_CUR_MEASURE = re.compile(r'\[\s*MEASURE\s*\]')
RE_GTEST_REF_MEASURE = re.compile(
r'Reference values of virtual memory consumption')
RE_GTEST_PASSED = re.compile(r'\[\s*PASSED\s*\]')
ref_metrics = dict(zip(heading, values))
for index in reversed(range(len(log_lines))):
if RE_GTEST_CUR_MEASURE.search(log_lines[index]):
+ test_name = log_lines[index].split()[-1]
heading = [name.lower() for name in log_lines[index+1]
[len(GTEST_INFO):].split()]
values = [int(val) for val in log_lines[index+2]
[len(GTEST_INFO):].split()]
entry = SimpleNamespace(
metrics=dict(zip(heading, values)),
- test_name=model['test'],
+ test_name=test_name,
model_name=os.path.splitext(
os.path.basename(model['path']))[0],
precision=next(pr for pr in PRECISSIONS if pr.upper()
projects / platform / upstream / dldt.git / commitdiff