modules/ts/src/ts_tags.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 #include "precomp.hpp"
   6
   7 #include "ts_tags.hpp"
   8
   9 namespace cvtest {
  10
  11 static bool printTestTag = false;
  12
  13 static std::vector<std::string> currentDirectTestTags, currentImpliedTestTags;
  14 static std::vector<const ::testing::TestInfo*> skipped_tests;
  15
  16 static std::map<std::string, int>& getTestTagsSkipCounts()
  17 {
  18     static std::map<std::string, int> testTagsSkipCounts;
  19     return testTagsSkipCounts;
  20 }
  21 static std::map<std::string, int>& getTestTagsSkipExtraCounts()
  22 {
  23     static std::map<std::string, int> testTagsSkipExtraCounts;
  24     return testTagsSkipExtraCounts;
  25 }
  26 static void increaseTagsSkipCount(const std::string& tag, bool isMain)
  27 {
  28     std::map<std::string, int>& counts = isMain ? getTestTagsSkipCounts() : getTestTagsSkipExtraCounts();
  29     std::map<std::string, int>::iterator i = counts.find(tag);
  30     if (i == counts.end())
  31     {
  32         counts[tag] = 1;
  33     }
  34     else
  35     {
  36         i->second++;
  37     }
  38 }
  39
  40 static std::vector<std::string>& getTestTagsSkipList()
  41 {
  42     static std::vector<std::string> testSkipWithTags;
  43     static bool initialized = false;
  44     if (!initialized)
  45     {
  46 #if OPENCV_32BIT_CONFIGURATION
  47         testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_2GB);
  48 #else
  49         testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_6GB);
  50 #endif
  51         testSkipWithTags.push_back(CV_TEST_TAG_VERYLONG);
  52 #if defined(_DEBUG)
  53         testSkipWithTags.push_back(CV_TEST_TAG_DEBUG_VERYLONG);
  54 #endif
  55         initialized = true;
  56     }
  57     return testSkipWithTags;
  58 }
  59
  60 void registerGlobalSkipTag(const std::string& skipTag)
  61 {
  62     std::vector<std::string>& skipTags = getTestTagsSkipList();
  63     for (size_t i = 0; i < skipTags.size(); ++i)
  64     {
  65         if (skipTag == skipTags[i])
  66             return;  // duplicate
  67     }
  68     skipTags.push_back(skipTag);
  69 }
  70
  71 static std::vector<std::string>& getTestTagsForceList()
  72 {
  73     static std::vector<std::string> getTestTagsForceList;
  74     return getTestTagsForceList;
  75 }
  76
  77 static std::vector<std::string>& getTestTagsRequiredList()
  78 {
  79     static std::vector<std::string> getTestTagsRequiredList;
  80     return getTestTagsRequiredList;
  81 }
  82
  83
  84 class TestTagsListener: public ::testing::EmptyTestEventListener
  85 {
  86 public:
  87     void OnTestProgramStart(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
  88     {
  89         {
  90             const std::vector<std::string>& tags = getTestTagsRequiredList();
  91             std::ostringstream os, os_direct;
  92             for (size_t i = 0; i < tags.size(); i++)
  93             {
  94                 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
  95                 os_direct << (i == 0 ? "" : ",") << tags[i];
  96             }
  97             std::string tags_str = os.str();
  98             if (!tags.empty())
  99                 std::cout << "TEST: Run tests with tags: " << tags_str << std::endl;
 100             ::testing::Test::RecordProperty("test_tags", os_direct.str());
 101         }
 102         {
 103             const std::vector<std::string>& tags = getTestTagsSkipList();
 104             std::ostringstream os, os_direct;
 105             for (size_t i = 0; i < tags.size(); i++)
 106             {
 107                 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
 108                 os_direct << (i == 0 ? "" : ",") << tags[i];
 109             }
 110             std::string tags_str = os.str();
 111             if (!tags.empty())
 112                 std::cout << "TEST: Skip tests with tags: " << tags_str << std::endl;
 113             ::testing::Test::RecordProperty("test_tags_skip", os_direct.str());
 114         }
 115         {
 116             const std::vector<std::string>& tags = getTestTagsForceList();
 117             std::ostringstream os, os_direct;
 118             for (size_t i = 0; i < tags.size(); i++)
 119             {
 120                 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
 121                 os_direct << (i == 0 ? "" : ",") << tags[i];
 122             }
 123             std::string tags_str = os.str();
 124             if (!tags.empty())
 125                 std::cout << "TEST: Force tests with tags: " << tags_str << std::endl;
 126             ::testing::Test::RecordProperty("test_tags_force", os_direct.str());
 127         }
 128     }
 129
 130     void OnTestStart(const ::testing::TestInfo& test_info) CV_OVERRIDE
 131     {
 132         currentDirectTestTags.clear();
 133         currentImpliedTestTags.clear();
 134
 135         const char* value_param_ = test_info.value_param();
 136         if (value_param_)
 137         {
 138             std::string value_param(value_param_);
 139             if (value_param.find("CV_64F") != std::string::npos
 140                 || (value_param.find("64F") != std::string::npos
 141                     && value_param.find(" 64F") != std::string::npos
 142                     && value_param.find(",64F") != std::string::npos
 143                     && value_param.find("(64F") != std::string::npos
 144                 )
 145             )
 146                 applyTestTag_(CV_TEST_TAG_TYPE_64F);
 147             if (value_param.find("1280x720") != std::string::npos)
 148                 applyTestTag_(CV_TEST_TAG_SIZE_HD);
 149             if (value_param.find("1920x1080") != std::string::npos)
 150                 applyTestTag_(CV_TEST_TAG_SIZE_FULLHD);
 151             if (value_param.find("3840x2160") != std::string::npos)
 152                 applyTestTag_(CV_TEST_TAG_SIZE_4K);
 153         }
 154     }
 155
 156     void OnTestEnd(const ::testing::TestInfo& /*test_info*/) CV_OVERRIDE
 157     {
 158         if (currentDirectTestTags.empty() && currentImpliedTestTags.empty())
 159         {
 160             if (printTestTag) std::cout << "[     TAGS ] No tags" << std::endl;
 161             return;
 162         }
 163         std::ostringstream os;
 164         std::ostringstream os_direct;
 165         std::ostringstream os_implied;
 166         {
 167             const std::vector<std::string>& tags = currentDirectTestTags;
 168             for (size_t i = 0; i < tags.size(); i++)
 169             {
 170                 os << (i == 0 ? "" : ", ") << tags[i];
 171                 os_direct << (i == 0 ? "" : ",") << tags[i];
 172             }
 173         }
 174         if (!currentImpliedTestTags.empty())
 175         {
 176             os << " (implied tags: ";
 177             const std::vector<std::string>& tags = currentImpliedTestTags;
 178             for (size_t i = 0; i < tags.size(); i++)
 179             {
 180                 os << (i == 0 ? "" : ", ") << tags[i];
 181                 os_implied << (i == 0 ? "" : ",") << tags[i];
 182             }
 183             os << ")";
 184         }
 185         if (printTestTag) std::cout << "[     TAGS ] " << os.str() << std::endl;
 186         ::testing::Test::RecordProperty("tags", os_direct.str());
 187         ::testing::Test::RecordProperty("tags_implied", os_implied.str());
 188     }
 189
 190     void OnTestIterationEnd(const ::testing::UnitTest& /*unit_test*/, int /*iteration*/) CV_OVERRIDE
 191     {
 192         if (!skipped_tests.empty())
 193         {
 194             std::cout << "[ SKIPSTAT ] " << skipped_tests.size() << " tests via tags" << std::endl;
 195             const std::vector<std::string>& skipTags = getTestTagsSkipList();
 196             const std::map<std::string, int>& counts = getTestTagsSkipCounts();
 197             const std::map<std::string, int>& countsExtra = getTestTagsSkipExtraCounts();
 198             for (std::vector<std::string>::const_iterator i = skipTags.begin(); i != skipTags.end(); ++i)
 199             {
 200                 int c1 = 0;
 201                 std::map<std::string, int>::const_iterator i1 = counts.find(*i);
 202                 if (i1 != counts.end()) c1 = i1->second;
 203                 int c2 = 0;
 204                 std::map<std::string, int>::const_iterator i2 = countsExtra.find(*i);
 205                 if (i2 != countsExtra.end()) c2 = i2->second;
 206                 if (c2 > 0)
 207                 {
 208                     std::cout << "[ SKIPSTAT ] TAG='" << *i << "' skip " << c1 << " tests (" << c2 << " times in extra skip list)" << std::endl;
 209                 }
 210                 else if (c1 > 0)
 211                 {
 212                     std::cout << "[ SKIPSTAT ] TAG='" << *i << "' skip " << c1 << " tests" << std::endl;
 213                 }
 214             }
 215         }
 216         skipped_tests.clear();
 217     }
 218
 219     void OnTestProgramEnd(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
 220     {
 221         /*if (!skipped_tests.empty())
 222         {
 223             for (size_t i = 0; i < skipped_tests.size(); i++)
 224             {
 225                 const ::testing::TestInfo* test_info = skipped_tests[i];
 226                 if (!test_info) continue;
 227                 std::cout << "- " << test_info->test_case_name() << "." << test_info->name() << std::endl;
 228             }
 229         }*/
 230     }
 231 };
 232
 233 static bool isTestTagForced(const std::string& testTag)
 234 {
 235     const std::vector<std::string>& forceTags = getTestTagsForceList();
 236     for (size_t i = 0; i < forceTags.size(); ++i)
 237     {
 238         const std::string& forceTag = forceTags[i];
 239         if (testTag == forceTag
 240             || (testTag.size() >= forceTag.size()
 241                 && forceTag[forceTag.size() - 1] == '*'
 242                 && forceTag.substr(0, forceTag.size() - 1) == testTag.substr(0, forceTag.size() - 1)
 243             )
 244         )
 245         {
 246             return true;
 247         }
 248     }
 249     return false;
 250 }
 251
 252 static bool isTestTagSkipped(const std::string& testTag, CV_OUT std::string& skippedByTag)
 253 {
 254     skippedByTag.clear();
 255     const std::vector<std::string>& skipTags = getTestTagsSkipList();
 256     for (size_t i = 0; i < skipTags.size(); ++i)
 257     {
 258         const std::string& skipTag = skipTags[i];
 259         if (testTag == skipTag
 260             || (testTag.size() >= skipTag.size()
 261                 && skipTag[skipTag.size() - 1] == '*'
 262                 && skipTag.substr(0, skipTag.size() - 1) == testTag.substr(0, skipTag.size() - 1)
 263             )
 264         )
 265         {
 266             skippedByTag = skipTag;
 267             return true;
 268         }
 269     }
 270     return false;
 271 }
 272
 273 void checkTestTags()
 274 {
 275     std::string skipTag;
 276     const std::vector<std::string>& testTags = currentDirectTestTags;
 277     {
 278         const std::vector<std::string>& tags = getTestTagsRequiredList();
 279         if (!tags.empty())
 280         {
 281             size_t found = 0;
 282             for (size_t i = 0; i < tags.size(); ++i)
 283             {
 284                 const std::string& tag = tags[i];
 285                 for (size_t j = 0; j < testTags.size(); ++j)
 286                 {
 287                     const std::string& testTag = testTags[i];
 288                     if (testTag == tag
 289                         || (testTag.size() >= tag.size()
 290                             && tag[tag.size() - 1] == '*'
 291                             && tag.substr(0, tag.size() - 1) == testTag.substr(0, tag.size() - 1)
 292                         )
 293                     )
 294                     {
 295                         found++;
 296                         break;
 297                     }
 298                 }
 299             }
 300             if (found != tags.size())
 301             {
 302                 skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
 303                 throw SkipTestException("Test tags don't pass required tags list (--test_tag parameter)");
 304             }
 305         }
 306     }
 307     for (size_t i = 0; i < testTags.size(); ++i)
 308     {
 309         const std::string& testTag = testTags[i];
 310         if (isTestTagForced(testTag))
 311             return;
 312     }
 313     std::string skip_message;
 314     for (size_t i = 0; i < testTags.size(); ++i)
 315     {
 316         const std::string& testTag = testTags[i];
 317         if (isTestTagSkipped(testTag, skipTag))
 318         {
 319             increaseTagsSkipCount(skipTag, skip_message.empty());
 320             if (skip_message.empty()) skip_message = "Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)";
 321         }
 322     }
 323     const std::vector<std::string>& testTagsImplied = currentImpliedTestTags;
 324     for (size_t i = 0; i < testTagsImplied.size(); ++i)
 325     {
 326         const std::string& testTag = testTagsImplied[i];
 327         if (isTestTagSkipped(testTag, skipTag))
 328         {
 329             increaseTagsSkipCount(skipTag, skip_message.empty());
 330             if (skip_message.empty()) skip_message = "Test with tag '" + testTag + "' is skipped (implied '" + skipTag + "' is in skip list)";
 331         }
 332     }
 333
 334     if (!skip_message.empty())
 335     {
 336         skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
 337         throw SkipTestException(skip_message);
 338     }
 339 }
 340
 341 static bool applyTestTagImpl(const std::string& tag, bool direct = false)
 342 {
 343     CV_Assert(!tag.empty());
 344     std::vector<std::string>& testTags = direct ? currentDirectTestTags : currentImpliedTestTags;
 345     for (size_t i = 0; i < testTags.size(); ++i)
 346     {
 347         const std::string& testTag = testTags[i];
 348         if (tag == testTag)
 349         {
 350             return false;  // already exists, skip
 351         }
 352     }
 353     testTags.push_back(tag);
 354
 355     // Tags implies logic
 356     if (tag == CV_TEST_TAG_MEMORY_14GB)
 357         applyTestTagImpl(CV_TEST_TAG_MEMORY_6GB);
 358     if (tag == CV_TEST_TAG_MEMORY_6GB)
 359         applyTestTagImpl(CV_TEST_TAG_MEMORY_2GB);
 360     if (tag == CV_TEST_TAG_MEMORY_2GB)
 361         applyTestTagImpl(CV_TEST_TAG_MEMORY_1GB);
 362     if (tag == CV_TEST_TAG_MEMORY_1GB)
 363         applyTestTagImpl(CV_TEST_TAG_MEMORY_512MB);
 364     if (tag == CV_TEST_TAG_VERYLONG)
 365     {
 366         applyTestTagImpl(CV_TEST_TAG_DEBUG_VERYLONG);
 367         applyTestTagImpl(CV_TEST_TAG_LONG);
 368     }
 369     else if (tag == CV_TEST_TAG_DEBUG_VERYLONG)
 370     {
 371         applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
 372     }
 373     else if (tag == CV_TEST_TAG_LONG)
 374     {
 375         applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
 376     }
 377
 378     if (tag == CV_TEST_TAG_SIZE_4K)
 379         applyTestTagImpl(CV_TEST_TAG_SIZE_FULLHD);
 380     if (tag == CV_TEST_TAG_SIZE_FULLHD)
 381         applyTestTagImpl(CV_TEST_TAG_SIZE_HD);
 382
 383     return true;
 384 }
 385
 386 void applyTestTag(const std::string& tag)
 387 {
 388     if (tag.empty()) return;
 389     if (!applyTestTagImpl(tag, true))
 390         return;
 391     checkTestTags();
 392 }
 393
 394 void applyTestTag_(const std::string& tag)
 395 {
 396     if (tag.empty()) return;
 397     if (!applyTestTagImpl(tag, true))
 398         return;
 399 }
 400
 401 static std::vector<std::string> parseStringList(const std::string& s)
 402 {
 403     std::vector<std::string> result;
 404     size_t start_pos = 0;
 405     while (start_pos != std::string::npos)
 406     {
 407         while (start_pos < s.size() && s[start_pos] == ' ')
 408             start_pos++;
 409         const size_t pos_ = s.find(',', start_pos);
 410         size_t pos = (pos_ == std::string::npos ? s.size() : pos_);
 411         while (pos > start_pos && s[pos - 1] == ' ')
 412             pos--;
 413         if (pos > start_pos)
 414         {
 415             const std::string one_piece(s, start_pos, pos - start_pos);
 416             result.push_back(one_piece);
 417         }
 418         start_pos = (pos_ == std::string::npos ? pos_ : pos_ + 1);
 419     }
 420     return result;
 421
 422 }
 423
 424 void activateTestTags(const cv::CommandLineParser& parser)
 425 {
 426     std::string test_tag_skip = parser.get<std::string>("test_tag_skip");
 427     if (!test_tag_skip.empty())
 428     {
 429         const std::vector<std::string> tag_list = parseStringList(test_tag_skip);
 430         if (!tag_list.empty())
 431         {
 432             std::vector<std::string>& skipTags = getTestTagsSkipList();
 433             for (size_t k = 0; k < tag_list.size(); ++k)
 434             {
 435                 const std::string& tag = tag_list[k];
 436                 bool found = false;
 437                 for (size_t i = 0; i < skipTags.size(); ++i)
 438                 {
 439                     if (tag == skipTags[i])
 440                     {
 441                         found = true;
 442                         break;
 443                     }
 444                 }
 445                 if (!found)
 446                     skipTags.push_back(tag);
 447             }
 448         }
 449     }
 450
 451     std::string test_tag_enable = parser.get<std::string>("test_tag_enable");
 452     if (!test_tag_enable.empty())
 453     {
 454         const std::vector<std::string> tag_list = parseStringList(test_tag_enable);
 455         if (!tag_list.empty())
 456         {
 457             std::vector<std::string>& skipTags = getTestTagsSkipList();
 458             for (size_t k = 0; k < tag_list.size(); ++k)
 459             {
 460                 const std::string& tag = tag_list[k];
 461                 bool found = false;
 462                 for (size_t i = 0; i < skipTags.size(); ++i)
 463                 {
 464                     if (tag == skipTags[i])
 465                     {
 466                         skipTags.erase(skipTags.begin() + i);
 467                         found = true;
 468                     }
 469                 }
 470                 if (!found)
 471                 {
 472                     std::cerr << "Can't re-enable tag '" << tag << "' - it is not in the skip list" << std::endl;
 473                 }
 474             }
 475         }
 476     }
 477
 478     std::string test_tag_force = parser.get<std::string>("test_tag_force");
 479     if (!test_tag_force.empty())
 480     {
 481         const std::vector<std::string> tag_list = parseStringList(test_tag_force);
 482         if (!tag_list.empty())
 483         {
 484             std::vector<std::string>& forceTags = getTestTagsForceList();
 485             for (size_t k = 0; k < tag_list.size(); ++k)
 486             {
 487                 const std::string& tag = tag_list[k];
 488                 bool found = false;
 489                 for (size_t i = 0; i < forceTags.size(); ++i)
 490                 {
 491                     if (tag == forceTags[i])
 492                     {
 493                         found = true;
 494                         break;
 495                     }
 496                 }
 497                 if (!found)
 498                     forceTags.push_back(tag);
 499             }
 500         }
 501     }
 502
 503     std::string test_tag = parser.get<std::string>("test_tag");
 504     if (!test_tag.empty())
 505     {
 506         const std::vector<std::string> tag_list = parseStringList(test_tag);
 507         if (!tag_list.empty())
 508         {
 509             std::vector<std::string>& requiredTags = getTestTagsRequiredList();
 510             for (size_t k = 0; k < tag_list.size(); ++k)
 511             {
 512                 const std::string& tag = tag_list[k];
 513                 bool found = false;
 514                 for (size_t i = 0; i < requiredTags.size(); ++i)
 515                 {
 516                     if (tag == requiredTags[i])
 517                     {
 518                         found = true;
 519                         break;
 520                     }
 521                 }
 522                 if (!found)
 523                     requiredTags.push_back(tag);
 524             }
 525         }
 526     }
 527
 528     printTestTag = parser.get<bool>("test_tag_print");
 529
 530     ::testing::UnitTest::GetInstance()->listeners().Append(new TestTagsListener());
 531 }
 532
 533 } // namespace