modules/ts/src/ts_tags.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 #include "precomp.hpp"
   6
   7 #include "ts_tags.hpp"
   8
   9 namespace cvtest {
  10
  11 static bool printTestTag = false;
  12
  13 static std::vector<std::string> currentDirectTestTags, currentImpliedTestTags;
  14 static std::vector<const ::testing::TestInfo*> skipped_tests;
  15
  16 static std::vector<std::string>& getTestTagsSkipList()
  17 {
  18     static std::vector<std::string> testSkipWithTags;
  19     static bool initialized = false;
  20     if (!initialized)
  21     {
  22 #if OPENCV_32BIT_CONFIGURATION
  23         testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_2GB);
  24 #else
  25         testSkipWithTags.push_back(CV_TEST_TAG_MEMORY_6GB);
  26 #endif
  27         testSkipWithTags.push_back(CV_TEST_TAG_VERYLONG);
  28 #if defined(_DEBUG)
  29         testSkipWithTags.push_back(CV_TEST_TAG_DEBUG_VERYLONG);
  30 #endif
  31         initialized = true;
  32     }
  33     return testSkipWithTags;
  34 }
  35
  36 static std::vector<std::string>& getTestTagsForceList()
  37 {
  38     static std::vector<std::string> getTestTagsForceList;
  39     return getTestTagsForceList;
  40 }
  41
  42 static std::vector<std::string>& getTestTagsRequiredList()
  43 {
  44     static std::vector<std::string> getTestTagsRequiredList;
  45     return getTestTagsRequiredList;
  46 }
  47
  48
  49 class TestTagsListener: public ::testing::EmptyTestEventListener
  50 {
  51 public:
  52     void OnTestProgramStart(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
  53     {
  54         {
  55             const std::vector<std::string>& tags = getTestTagsRequiredList();
  56             std::ostringstream os, os_direct;
  57             for (size_t i = 0; i < tags.size(); i++)
  58             {
  59                 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
  60                 os_direct << (i == 0 ? "" : ",") << tags[i];
  61             }
  62             std::string tags_str = os.str();
  63             if (!tags.empty())
  64                 std::cout << "TEST: Run tests with tags: " << tags_str << std::endl;
  65             ::testing::Test::RecordProperty("test_tags", os_direct.str());
  66         }
  67         {
  68             const std::vector<std::string>& tags = getTestTagsSkipList();
  69             std::ostringstream os, os_direct;
  70             for (size_t i = 0; i < tags.size(); i++)
  71             {
  72                 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
  73                 os_direct << (i == 0 ? "" : ",") << tags[i];
  74             }
  75             std::string tags_str = os.str();
  76             if (!tags.empty())
  77                 std::cout << "TEST: Skip tests with tags: " << tags_str << std::endl;
  78             ::testing::Test::RecordProperty("test_tags_skip", os_direct.str());
  79         }
  80         {
  81             const std::vector<std::string>& tags = getTestTagsForceList();
  82             std::ostringstream os, os_direct;
  83             for (size_t i = 0; i < tags.size(); i++)
  84             {
  85                 os << (i == 0 ? "'" : ", '") << tags[i] << "'";
  86                 os_direct << (i == 0 ? "" : ",") << tags[i];
  87             }
  88             std::string tags_str = os.str();
  89             if (!tags.empty())
  90                 std::cout << "TEST: Force tests with tags: " << tags_str << std::endl;
  91             ::testing::Test::RecordProperty("test_tags_force", os_direct.str());
  92         }
  93     }
  94
  95     void OnTestStart(const ::testing::TestInfo& test_info) CV_OVERRIDE
  96     {
  97         currentDirectTestTags.clear();
  98         currentImpliedTestTags.clear();
  99
 100         const char* value_param_ = test_info.value_param();
 101         if (value_param_)
 102         {
 103             std::string value_param(value_param_);
 104             if (value_param.find("CV_64F") != std::string::npos
 105                 || (value_param.find("64F") != std::string::npos
 106                     && value_param.find(" 64F") != std::string::npos
 107                     && value_param.find(",64F") != std::string::npos
 108                     && value_param.find("(64F") != std::string::npos
 109                 )
 110             )
 111                 applyTestTag_(CV_TEST_TAG_TYPE_64F);
 112             if (value_param.find("1280x720") != std::string::npos)
 113                 applyTestTag_(CV_TEST_TAG_SIZE_HD);
 114             if (value_param.find("1920x1080") != std::string::npos)
 115                 applyTestTag_(CV_TEST_TAG_SIZE_FULLHD);
 116             if (value_param.find("3840x2160") != std::string::npos)
 117                 applyTestTag_(CV_TEST_TAG_SIZE_4K);
 118         }
 119     }
 120
 121     void OnTestEnd(const ::testing::TestInfo& /*test_info*/) CV_OVERRIDE
 122     {
 123         if (currentDirectTestTags.empty() && currentImpliedTestTags.empty())
 124         {
 125             if (printTestTag) std::cout << "[     TAGS ] No tags" << std::endl;
 126             return;
 127         }
 128         std::ostringstream os;
 129         std::ostringstream os_direct;
 130         std::ostringstream os_implied;
 131         {
 132             const std::vector<std::string>& tags = currentDirectTestTags;
 133             for (size_t i = 0; i < tags.size(); i++)
 134             {
 135                 os << (i == 0 ? "" : ", ") << tags[i];
 136                 os_direct << (i == 0 ? "" : ",") << tags[i];
 137             }
 138         }
 139         if (!currentImpliedTestTags.empty())
 140         {
 141             os << " (implied tags: ";
 142             const std::vector<std::string>& tags = currentImpliedTestTags;
 143             for (size_t i = 0; i < tags.size(); i++)
 144             {
 145                 os << (i == 0 ? "" : ", ") << tags[i];
 146                 os_implied << (i == 0 ? "" : ",") << tags[i];
 147             }
 148             os << ")";
 149         }
 150         if (printTestTag) std::cout << "[     TAGS ] " << os.str() << std::endl;
 151         ::testing::Test::RecordProperty("tags", os_direct.str());
 152         ::testing::Test::RecordProperty("tags_implied", os_implied.str());
 153     }
 154
 155     void OnTestIterationEnd(const ::testing::UnitTest& /*unit_test*/, int /*iteration*/) CV_OVERRIDE
 156     {
 157         if (!skipped_tests.empty())
 158         {
 159             std::cout << "[   SKIP   ] " << skipped_tests.size() << " tests via tags" << std::endl;
 160         }
 161         skipped_tests.clear();
 162     }
 163
 164     void OnTestProgramEnd(const ::testing::UnitTest& /*unit_test*/) CV_OVERRIDE
 165     {
 166         /*if (!skipped_tests.empty())
 167         {
 168             for (size_t i = 0; i < skipped_tests.size(); i++)
 169             {
 170                 const ::testing::TestInfo* test_info = skipped_tests[i];
 171                 if (!test_info) continue;
 172                 std::cout << "- " << test_info->test_case_name() << "." << test_info->name() << std::endl;
 173             }
 174         }*/
 175     }
 176 };
 177
 178 static bool isTestTagForced(const std::string& testTag)
 179 {
 180     const std::vector<std::string>& forceTags = getTestTagsForceList();
 181     for (size_t i = 0; i < forceTags.size(); ++i)
 182     {
 183         const std::string& forceTag = forceTags[i];
 184         if (testTag == forceTag
 185             || (testTag.size() >= forceTag.size()
 186                 && forceTag[forceTag.size() - 1] == '*'
 187                 && forceTag.substr(0, forceTag.size() - 1) == testTag.substr(0, forceTag.size() - 1)
 188             )
 189         )
 190         {
 191             return true;
 192         }
 193     }
 194     return false;
 195 }
 196
 197 static bool isTestTagSkipped(const std::string& testTag, CV_OUT std::string& skippedByTag)
 198 {
 199     skippedByTag.clear();
 200     const std::vector<std::string>& skipTags = getTestTagsSkipList();
 201     for (size_t i = 0; i < skipTags.size(); ++i)
 202     {
 203         const std::string& skipTag = skipTags[i];
 204         if (testTag == skipTag
 205             || (testTag.size() >= skipTag.size()
 206                 && skipTag[skipTag.size() - 1] == '*'
 207                 && skipTag.substr(0, skipTag.size() - 1) == testTag.substr(0, skipTag.size() - 1)
 208             )
 209         )
 210         {
 211             skippedByTag = skipTag;
 212             return true;
 213         }
 214     }
 215     return false;
 216 }
 217
 218 void checkTestTags()
 219 {
 220     std::string skipTag;
 221     const std::vector<std::string>& testTags = currentDirectTestTags;
 222     {
 223         const std::vector<std::string>& tags = getTestTagsRequiredList();
 224         if (!tags.empty())
 225         {
 226             size_t found = 0;
 227             for (size_t i = 0; i < tags.size(); ++i)
 228             {
 229                 const std::string& tag = tags[i];
 230                 for (size_t j = 0; j < testTags.size(); ++j)
 231                 {
 232                     const std::string& testTag = testTags[i];
 233                     if (testTag == tag
 234                         || (testTag.size() >= tag.size()
 235                             && tag[tag.size() - 1] == '*'
 236                             && tag.substr(0, tag.size() - 1) == testTag.substr(0, tag.size() - 1)
 237                         )
 238                     )
 239                     {
 240                         found++;
 241                         break;
 242                     }
 243                 }
 244             }
 245             if (found != tags.size())
 246             {
 247                 skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
 248                 throw SkipTestException("Test tags don't pass required tags list (--test_tag parameter)");
 249             }
 250         }
 251     }
 252     for (size_t i = 0; i < testTags.size(); ++i)
 253     {
 254         const std::string& testTag = testTags[i];
 255         if (isTestTagForced(testTag))
 256             return;
 257     }
 258     for (size_t i = 0; i < testTags.size(); ++i)
 259     {
 260         const std::string& testTag = testTags[i];
 261         if (isTestTagSkipped(testTag, skipTag))
 262         {
 263             skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
 264             throw SkipTestException("Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)");
 265         }
 266     }
 267     const std::vector<std::string>& testTagsImplied = currentImpliedTestTags;
 268     for (size_t i = 0; i < testTagsImplied.size(); ++i)
 269     {
 270         const std::string& testTag = testTagsImplied[i];
 271         if (isTestTagSkipped(testTag, skipTag))
 272         {
 273             skipped_tests.push_back(::testing::UnitTest::GetInstance()->current_test_info());
 274             throw SkipTestException("Test with tag '" + testTag + "' is skipped ('" + skipTag + "' is in skip list)");
 275         }
 276     }
 277 }
 278
 279 static bool applyTestTagImpl(const std::string& tag, bool direct = false)
 280 {
 281     CV_Assert(!tag.empty());
 282     std::vector<std::string>& testTags = direct ? currentDirectTestTags : currentImpliedTestTags;
 283     for (size_t i = 0; i < testTags.size(); ++i)
 284     {
 285         const std::string& testTag = testTags[i];
 286         if (tag == testTag)
 287         {
 288             return false;  // already exists, skip
 289         }
 290     }
 291     testTags.push_back(tag);
 292
 293     // Tags implies logic
 294     if (tag == CV_TEST_TAG_MEMORY_14GB)
 295         applyTestTagImpl(CV_TEST_TAG_MEMORY_6GB);
 296     if (tag == CV_TEST_TAG_MEMORY_6GB)
 297         applyTestTagImpl(CV_TEST_TAG_MEMORY_2GB);
 298     if (tag == CV_TEST_TAG_MEMORY_2GB)
 299         applyTestTagImpl(CV_TEST_TAG_MEMORY_1GB);
 300     if (tag == CV_TEST_TAG_MEMORY_1GB)
 301         applyTestTagImpl(CV_TEST_TAG_MEMORY_512MB);
 302     if (tag == CV_TEST_TAG_VERYLONG)
 303     {
 304         applyTestTagImpl(CV_TEST_TAG_DEBUG_VERYLONG);
 305         applyTestTagImpl(CV_TEST_TAG_LONG);
 306     }
 307     else if (tag == CV_TEST_TAG_DEBUG_VERYLONG)
 308     {
 309         applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
 310     }
 311     else if (tag == CV_TEST_TAG_LONG)
 312     {
 313         applyTestTagImpl(CV_TEST_TAG_DEBUG_LONG);
 314     }
 315
 316     if (tag == CV_TEST_TAG_SIZE_4K)
 317         applyTestTagImpl(CV_TEST_TAG_SIZE_FULLHD);
 318     if (tag == CV_TEST_TAG_SIZE_FULLHD)
 319         applyTestTagImpl(CV_TEST_TAG_SIZE_HD);
 320
 321     return true;
 322 }
 323
 324 void applyTestTag(const std::string& tag)
 325 {
 326     if (tag.empty()) return;
 327     if (!applyTestTagImpl(tag, true))
 328         return;
 329     checkTestTags();
 330 }
 331
 332 void applyTestTag_(const std::string& tag)
 333 {
 334     if (tag.empty()) return;
 335     if (!applyTestTagImpl(tag, true))
 336         return;
 337 }
 338
 339 static std::vector<std::string> parseStringList(const std::string& s)
 340 {
 341     std::vector<std::string> result;
 342     size_t start_pos = 0;
 343     while (start_pos != std::string::npos)
 344     {
 345         while (start_pos < s.size() && s[start_pos] == ' ')
 346             start_pos++;
 347         const size_t pos_ = s.find(',', start_pos);
 348         size_t pos = (pos_ == std::string::npos ? s.size() : pos_);
 349         while (pos > start_pos && s[pos - 1] == ' ')
 350             pos--;
 351         if (pos > start_pos)
 352         {
 353             const std::string one_piece(s, start_pos, pos - start_pos);
 354             result.push_back(one_piece);
 355         }
 356         start_pos = (pos_ == std::string::npos ? pos_ : pos_ + 1);
 357     }
 358     return result;
 359
 360 }
 361
 362 void activateTestTags(const cv::CommandLineParser& parser)
 363 {
 364     std::string test_tag_skip = parser.get<std::string>("test_tag_skip");
 365     if (!test_tag_skip.empty())
 366     {
 367         const std::vector<std::string> tag_list = parseStringList(test_tag_skip);
 368         if (!tag_list.empty())
 369         {
 370             std::vector<std::string>& skipTags = getTestTagsSkipList();
 371             for (size_t k = 0; k < tag_list.size(); ++k)
 372             {
 373                 const std::string& tag = tag_list[k];
 374                 bool found = false;
 375                 for (size_t i = 0; i < skipTags.size(); ++i)
 376                 {
 377                     if (tag == skipTags[i])
 378                     {
 379                         found = true;
 380                         break;
 381                     }
 382                 }
 383                 if (!found)
 384                     skipTags.push_back(tag);
 385             }
 386         }
 387     }
 388
 389     std::string test_tag_enable = parser.get<std::string>("test_tag_enable");
 390     if (!test_tag_enable.empty())
 391     {
 392         const std::vector<std::string> tag_list = parseStringList(test_tag_enable);
 393         if (!tag_list.empty())
 394         {
 395             std::vector<std::string>& skipTags = getTestTagsSkipList();
 396             for (size_t k = 0; k < tag_list.size(); ++k)
 397             {
 398                 const std::string& tag = tag_list[k];
 399                 bool found = false;
 400                 for (size_t i = 0; i < skipTags.size(); ++i)
 401                 {
 402                     if (tag == skipTags[i])
 403                     {
 404                         skipTags.erase(skipTags.begin() + i);
 405                         found = true;
 406                     }
 407                 }
 408                 if (!found)
 409                 {
 410                     std::cerr << "Can't re-enable tag '" << tag << "' - it is not in the skip list" << std::endl;
 411                 }
 412             }
 413         }
 414     }
 415
 416     std::string test_tag_force = parser.get<std::string>("test_tag_force");
 417     if (!test_tag_force.empty())
 418     {
 419         const std::vector<std::string> tag_list = parseStringList(test_tag_force);
 420         if (!tag_list.empty())
 421         {
 422             std::vector<std::string>& forceTags = getTestTagsForceList();
 423             for (size_t k = 0; k < tag_list.size(); ++k)
 424             {
 425                 const std::string& tag = tag_list[k];
 426                 bool found = false;
 427                 for (size_t i = 0; i < forceTags.size(); ++i)
 428                 {
 429                     if (tag == forceTags[i])
 430                     {
 431                         found = true;
 432                         break;
 433                     }
 434                 }
 435                 if (!found)
 436                     forceTags.push_back(tag);
 437             }
 438         }
 439     }
 440
 441     std::string test_tag = parser.get<std::string>("test_tag");
 442     if (!test_tag.empty())
 443     {
 444         const std::vector<std::string> tag_list = parseStringList(test_tag);
 445         if (!tag_list.empty())
 446         {
 447             std::vector<std::string>& requiredTags = getTestTagsRequiredList();
 448             for (size_t k = 0; k < tag_list.size(); ++k)
 449             {
 450                 const std::string& tag = tag_list[k];
 451                 bool found = false;
 452                 for (size_t i = 0; i < requiredTags.size(); ++i)
 453                 {
 454                     if (tag == requiredTags[i])
 455                     {
 456                         found = true;
 457                         break;
 458                     }
 459                 }
 460                 if (!found)
 461                     requiredTags.push_back(tag);
 462             }
 463         }
 464     }
 465
 466     printTestTag = parser.get<bool>("test_tag_print");
 467
 468     ::testing::UnitTest::GetInstance()->listeners().Append(new TestTagsListener());
 469 }
 470
 471 } // namespace