1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
11 static bool printTestTag = false;
13 static std::vector<std::string> currentDirectTestTags, currentImpliedTestTags;
14 static std::vector<const ::testing::TestInfo*> skipped_tests;
16 static std::map<std::string, int>& getTestTagsSkipCounts()
18 static std::map<std::string, int> testTagsSkipCounts;
19 return testTagsSkipCounts;
21 static std::map<std::string, int>& getTestTagsSkipExtraCounts()
23 static std::map<std::string, int> testTagsSkipExtraCounts;
24 return testTagsSkipExtraCounts;
26 static void increaseTagsSkipCount(const std::string& tag, bool isMain)
28 std::map<std::string, int>& counts = isMain ? getTestTagsSkipCounts() : getTestTagsSkipExtraCounts();
29 std::map<std::string, int>::iterator i = counts.find(tag);
30 if (i == counts.end())
40 static std::vector<std::string>& getTestTagsSkipList()
42 static std::vector<std::string> testSkipWithTags;
43 static bool initialized = false;
46 #if OPENCV_32BIT_CONFIGURATION
47 testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_2GB);
49 testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_6GB);
51 testSkipWithTags.push_back(CV_TEST_TAG_VERYLONG);
53 testSkipWithTags.push_back(CV_TEST_TAG_DEBUG_VERYLONG);
57 return testSkipWithTags;
60 void registerGlobalSkipTag(const std::string& skipTag)
62 std::vector<std::string>& skipTags = getTestTagsSkipList();
63 for (size_t i = 0; i < skipTags.size(); ++i)
65 if (skipTag == skipTags[i])
68 skipTags.push_back(skipTag);
71 static std::vector<std::string>& getTestTagsForceList()
73 static std::vector<std::string> getTestTagsForceList;
74 return getTestTagsForceList;
77 static std::vector<std::string>& getTestTagsRequiredList()
79 static std::vector<std::string> getTestTagsRequiredList;
80 return getTestTagsRequiredList;
84 class TestTagsListener: public ::testing::EmptyTestEventListener
87 void OnTestProgramStart(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
90 const std::vector<std::string>& tags = getTestTagsRequiredList();
91 std::ostringstream os, os_direct;
92 for (size_t i = 0; i < tags.size(); i++)
94 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
95 os_direct << (i == 0 ? "" : ",") << tags[i];
97 std::string tags_str = os.str();
99 std::cout << "TEST: Run tests with tags: " << tags_str << std::endl;
100 ::testing::Test::RecordProperty("test_tags", os_direct.str());
103 const std::vector<std::string>& tags = getTestTagsSkipList();
104 std::ostringstream os, os_direct;
105 for (size_t i = 0; i < tags.size(); i++)
107 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
108 os_direct << (i == 0 ? "" : ",") << tags[i];
110 std::string tags_str = os.str();
112 std::cout << "TEST: Skip tests with tags: " << tags_str << std::endl;
113 ::testing::Test::RecordProperty("test_tags_skip", os_direct.str());
116 const std::vector<std::string>& tags = getTestTagsForceList();
117 std::ostringstream os, os_direct;
118 for (size_t i = 0; i < tags.size(); i++)
120 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
121 os_direct << (i == 0 ? "" : ",") << tags[i];
123 std::string tags_str = os.str();
125 std::cout << "TEST: Force tests with tags: " << tags_str << std::endl;
126 ::testing::Test::RecordProperty("test_tags_force", os_direct.str());
130 void OnTestStart(const ::testing::TestInfo& test_info) CV_OVERRIDE
132 currentDirectTestTags.clear();
133 currentImpliedTestTags.clear();
135 const char* value_param_ = test_info.value_param();
138 std::string value_param(value_param_);
139 if (value_param.find("CV_64F") != std::string::npos
140 || (value_param.find("64F") != std::string::npos
141 && value_param.find(" 64F") != std::string::npos
142 && value_param.find(",64F") != std::string::npos
143 && value_param.find("(64F") != std::string::npos
146 applyTestTag_(CV_TEST_TAG_TYPE_64F);
147 if (value_param.find("1280x720") != std::string::npos)
148 applyTestTag_(CV_TEST_TAG_SIZE_HD);
149 if (value_param.find("1920x1080") != std::string::npos)
150 applyTestTag_(CV_TEST_TAG_SIZE_FULLHD);
151 if (value_param.find("3840x2160") != std::string::npos)
152 applyTestTag_(CV_TEST_TAG_SIZE_4K);
156 void OnTestEnd(const ::testing::TestInfo& /*test_info*/) CV_OVERRIDE
158 if (currentDirectTestTags.empty() && currentImpliedTestTags.empty())
160 if (printTestTag) std::cout << "[ TAGS ] No tags" << std::endl;
163 std::ostringstream os;
164 std::ostringstream os_direct;
165 std::ostringstream os_implied;
167 const std::vector<std::string>& tags = currentDirectTestTags;
168 for (size_t i = 0; i < tags.size(); i++)
170 os << (i == 0 ? "" : ", ") << tags[i];
171 os_direct << (i == 0 ? "" : ",") << tags[i];
174 if (!currentImpliedTestTags.empty())
176 os << " (implied tags: ";
177 const std::vector<std::string>& tags = currentImpliedTestTags;
178 for (size_t i = 0; i < tags.size(); i++)
180 os << (i == 0 ? "" : ", ") << tags[i];
181 os_implied << (i == 0 ? "" : ",") << tags[i];
185 if (printTestTag) std::cout << "[ TAGS ] " << os.str() << std::endl;
186 ::testing::Test::RecordProperty("tags", os_direct.str());
187 ::testing::Test::RecordProperty("tags_implied", os_implied.str());
190 void OnTestIterationEnd(const ::testing::UnitTest& /*unit_test*/, int /*iteration*/) CV_OVERRIDE
192 if (!skipped_tests.empty())
194 std::cout << "[ SKIPSTAT ] " << skipped_tests.size() << " tests via tags" << std::endl;
195 const std::vector<std::string>& skipTags = getTestTagsSkipList();
196 const std::map<std::string, int>& counts = getTestTagsSkipCounts();
197 const std::map<std::string, int>& countsExtra = getTestTagsSkipExtraCounts();
198 for (std::vector<std::string>::const_iterator i = skipTags.begin(); i != skipTags.end(); ++i)
201 std::map<std::string, int>::const_iterator i1 = counts.find(*i);
202 if (i1 != counts.end()) c1 = i1->second;
204 std::map<std::string, int>::const_iterator i2 = countsExtra.find(*i);
205 if (i2 != countsExtra.end()) c2 = i2->second;
208 std::cout << "[ SKIPSTAT ] TAG='" << *i << "' skip " << c1 << " tests (" << c2 << " times in extra skip list)" << std::endl;
212 std::cout << "[ SKIPSTAT ] TAG='" << *i << "' skip " << c1 << " tests" << std::endl;
216 skipped_tests.clear();
219 void OnTestProgramEnd(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
221 /*if (!skipped_tests.empty())
223 for (size_t i = 0; i < skipped_tests.size(); i++)
225 const ::testing::TestInfo* test_info = skipped_tests[i];
226 if (!test_info) continue;
227 std::cout << "- " << test_info->test_case_name() << "." << test_info->name() << std::endl;
233 static bool isTestTagForced(const std::string& testTag)
235 const std::vector<std::string>& forceTags = getTestTagsForceList();
236 for (size_t i = 0; i < forceTags.size(); ++i)
238 const std::string& forceTag = forceTags[i];
239 if (testTag == forceTag
240 || (testTag.size() >= forceTag.size()
241 && forceTag[forceTag.size() - 1] == '*'
242 && forceTag.substr(0, forceTag.size() - 1) == testTag.substr(0, forceTag.size() - 1)
252 static bool isTestTagSkipped(const std::string& testTag, CV_OUT std::string& skippedByTag)
254 skippedByTag.clear();
255 const std::vector<std::string>& skipTags = getTestTagsSkipList();
256 for (size_t i = 0; i < skipTags.size(); ++i)
258 const std::string& skipTag = skipTags[i];
259 if (testTag == skipTag
260 || (testTag.size() >= skipTag.size()
261 && skipTag[skipTag.size() - 1] == '*'
262 && skipTag.substr(0, skipTag.size() - 1) == testTag.substr(0, skipTag.size() - 1)
266 skippedByTag = skipTag;
276 const std::vector<std::string>& testTags = currentDirectTestTags;
278 const std::vector<std::string>& tags = getTestTagsRequiredList();
282 for (size_t i = 0; i < tags.size(); ++i)
284 const std::string& tag = tags[i];
285 for (size_t j = 0; j < testTags.size(); ++j)
287 const std::string& testTag = testTags[i];
289 || (testTag.size() >= tag.size()
290 && tag[tag.size() - 1] == '*'
291 && tag.substr(0, tag.size() - 1) == testTag.substr(0, tag.size() - 1)
300 if (found != tags.size())
302 skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
303 throw SkipTestException("Test tags don't pass required tags list (--test_tag parameter)");
307 for (size_t i = 0; i < testTags.size(); ++i)
309 const std::string& testTag = testTags[i];
310 if (isTestTagForced(testTag))
313 std::string skip_message;
314 for (size_t i = 0; i < testTags.size(); ++i)
316 const std::string& testTag = testTags[i];
317 if (isTestTagSkipped(testTag, skipTag))
319 increaseTagsSkipCount(skipTag, skip_message.empty());
320 if (skip_message.empty()) skip_message = "Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)";
323 const std::vector<std::string>& testTagsImplied = currentImpliedTestTags;
324 for (size_t i = 0; i < testTagsImplied.size(); ++i)
326 const std::string& testTag = testTagsImplied[i];
327 if (isTestTagSkipped(testTag, skipTag))
329 increaseTagsSkipCount(skipTag, skip_message.empty());
330 if (skip_message.empty()) skip_message = "Test with tag '" + testTag + "' is skipped (implied '" + skipTag + "' is in skip list)";
334 if (!skip_message.empty())
336 skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
337 throw SkipTestException(skip_message);
341 static bool applyTestTagImpl(const std::string& tag, bool direct = false)
343 CV_Assert(!tag.empty());
344 std::vector<std::string>& testTags = direct ? currentDirectTestTags : currentImpliedTestTags;
345 for (size_t i = 0; i < testTags.size(); ++i)
347 const std::string& testTag = testTags[i];
350 return false; // already exists, skip
353 testTags.push_back(tag);
355 // Tags implies logic
356 if (tag == CV_TEST_TAG_MEMORY_14GB)
357 applyTestTagImpl(CV_TEST_TAG_MEMORY_6GB);
358 if (tag == CV_TEST_TAG_MEMORY_6GB)
359 applyTestTagImpl(CV_TEST_TAG_MEMORY_2GB);
360 if (tag == CV_TEST_TAG_MEMORY_2GB)
361 applyTestTagImpl(CV_TEST_TAG_MEMORY_1GB);
362 if (tag == CV_TEST_TAG_MEMORY_1GB)
363 applyTestTagImpl(CV_TEST_TAG_MEMORY_512MB);
364 if (tag == CV_TEST_TAG_VERYLONG)
366 applyTestTagImpl(CV_TEST_TAG_DEBUG_VERYLONG);
367 applyTestTagImpl(CV_TEST_TAG_LONG);
369 else if (tag == CV_TEST_TAG_DEBUG_VERYLONG)
371 applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
373 else if (tag == CV_TEST_TAG_LONG)
375 applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
378 if (tag == CV_TEST_TAG_SIZE_4K)
379 applyTestTagImpl(CV_TEST_TAG_SIZE_FULLHD);
380 if (tag == CV_TEST_TAG_SIZE_FULLHD)
381 applyTestTagImpl(CV_TEST_TAG_SIZE_HD);
386 void applyTestTag(const std::string& tag)
388 if (tag.empty()) return;
389 if (!applyTestTagImpl(tag, true))
394 void applyTestTag_(const std::string& tag)
396 if (tag.empty()) return;
397 if (!applyTestTagImpl(tag, true))
401 static std::vector<std::string> parseStringList(const std::string& s)
403 std::vector<std::string> result;
404 size_t start_pos = 0;
405 while (start_pos != std::string::npos)
407 while (start_pos < s.size() && s[start_pos] == ' ')
409 const size_t pos_ = s.find(',', start_pos);
410 size_t pos = (pos_ == std::string::npos ? s.size() : pos_);
411 while (pos > start_pos && s[pos - 1] == ' ')
415 const std::string one_piece(s, start_pos, pos - start_pos);
416 result.push_back(one_piece);
418 start_pos = (pos_ == std::string::npos ? pos_ : pos_ + 1);
424 void activateTestTags(const cv::CommandLineParser& parser)
426 std::string test_tag_skip = parser.get<std::string>("test_tag_skip");
427 if (!test_tag_skip.empty())
429 const std::vector<std::string> tag_list = parseStringList(test_tag_skip);
430 if (!tag_list.empty())
432 std::vector<std::string>& skipTags = getTestTagsSkipList();
433 for (size_t k = 0; k < tag_list.size(); ++k)
435 const std::string& tag = tag_list[k];
437 for (size_t i = 0; i < skipTags.size(); ++i)
439 if (tag == skipTags[i])
446 skipTags.push_back(tag);
451 std::string test_tag_enable = parser.get<std::string>("test_tag_enable");
452 if (!test_tag_enable.empty())
454 const std::vector<std::string> tag_list = parseStringList(test_tag_enable);
455 if (!tag_list.empty())
457 std::vector<std::string>& skipTags = getTestTagsSkipList();
458 for (size_t k = 0; k < tag_list.size(); ++k)
460 const std::string& tag = tag_list[k];
462 for (size_t i = 0; i < skipTags.size(); ++i)
464 if (tag == skipTags[i])
466 skipTags.erase(skipTags.begin() + i);
472 std::cerr << "Can't re-enable tag '" << tag << "' - it is not in the skip list" << std::endl;
478 std::string test_tag_force = parser.get<std::string>("test_tag_force");
479 if (!test_tag_force.empty())
481 const std::vector<std::string> tag_list = parseStringList(test_tag_force);
482 if (!tag_list.empty())
484 std::vector<std::string>& forceTags = getTestTagsForceList();
485 for (size_t k = 0; k < tag_list.size(); ++k)
487 const std::string& tag = tag_list[k];
489 for (size_t i = 0; i < forceTags.size(); ++i)
491 if (tag == forceTags[i])
498 forceTags.push_back(tag);
503 std::string test_tag = parser.get<std::string>("test_tag");
504 if (!test_tag.empty())
506 const std::vector<std::string> tag_list = parseStringList(test_tag);
507 if (!tag_list.empty())
509 std::vector<std::string>& requiredTags = getTestTagsRequiredList();
510 for (size_t k = 0; k < tag_list.size(); ++k)
512 const std::string& tag = tag_list[k];
514 for (size_t i = 0; i < requiredTags.size(); ++i)
516 if (tag == requiredTags[i])
523 requiredTags.push_back(tag);
528 printTestTag = parser.get<bool>("test_tag_print");
530 ::testing::UnitTest::GetInstance()->listeners().Append(new TestTagsListener());