libs/test/test/framework-ts/log-count-skipped-test.cpp

   1 //  (C) Copyright Raffi Enficiaud 2019.
   2 //  Distributed under the Boost Software License, Version 1.0.
   3 //  (See accompanying file LICENSE_1_0.txt or copy at
   4 //  http://www.boost.org/LICENSE_1_0.txt)
   5
   6 //  See http://www.boost.org/libs/test for the library home page.
   7 //
   8 // ***************************************************************************
   9
  10 // Boost.Test
  11
  12 #define BOOST_TEST_MODULE junit count skipped tests
  13 #include <boost/test/included/unit_test.hpp>
  14
  15 // our special logger for tests
  16 #include "logger-for-tests.hpp"
  17
  18 // STL
  19 #include <iostream>
  20 #include <ios>
  21
  22 using boost::test_tools::output_test_stream;
  23 using namespace boost::unit_test;
  24 namespace utf = boost::unit_test;
  25 namespace tt = boost::test_tools;
  26
  27
  28 struct skip_with_message
  29 {
  30     static bool default_enabled; // to control the status from outside
  31
  32     std::string message;
  33     skip_with_message(std::string m)
  34     : message(m) {}
  35
  36     tt::assertion_result operator()(utf::test_unit_id)
  37     {
  38         tt::assertion_result ans(skip_with_message::default_enabled);
  39         ans.message() << "test is skipped because " << message;
  40         return ans;
  41     }
  42 };
  43 bool skip_with_message::default_enabled = false;
  44
  45 void test_1()
  46 {
  47     BOOST_CHECK_EQUAL( 2 + 2, 4 );
  48 }
  49
  50 void test_2()
  51 {
  52     BOOST_CHECK_EQUAL( 0, 1 );
  53 }
  54
  55 void test_3()
  56 {
  57     BOOST_CHECK_EQUAL( 0, 1 );
  58 }
  59
  60 void check( output_test_stream& output,
  61             output_format log_format,
  62             test_unit_id id,
  63             log_level ll = log_successful_tests )
  64 {
  65     boost::unit_test::unit_test_log.set_format(log_format);
  66     boost::unit_test::unit_test_log.set_stream(output);
  67     boost::unit_test::unit_test_log.set_threshold_level(ll);
  68
  69     class reset_status : public test_tree_visitor {
  70     private:
  71         virtual bool    visit( test_unit const& tu )
  72         {
  73             const_cast<test_unit&>(tu).p_default_status.value = test_case::RS_INHERIT;
  74             const_cast<test_unit&>(tu).p_run_status.value = test_case::RS_INVALID;
  75             return true;
  76         }
  77     } rstatus;
  78
  79     // reinit the default/run status otherwise we cannot apply the decorators
  80     // after the first run
  81     traverse_test_tree( id, rstatus, true );
  82     framework::get<test_suite>(id).p_default_status.value = test_unit::RS_ENABLED;
  83
  84     output << "* " << log_format << "-format  *******************************************************************";
  85     output << std::endl;
  86
  87     framework::finalize_setup_phase( id );
  88     framework::run( id, false ); // do not continue the test tree to have the test_log_start/end
  89     output << std::endl;
  90
  91     boost::unit_test::unit_test_log.set_format(OF_CLF);
  92     boost::unit_test::unit_test_log.set_stream(std::cout);
  93
  94     BOOST_TEST( output.match_pattern(true) ); // flushes the stream at the end of the comparison.
  95 }
  96
  97 //____________________________________________________________________________//
  98
  99 void check( output_test_stream& output, test_suite* ts )
 100 {
 101     check( output, OF_CLF, ts->p_id );
 102     check( output, OF_XML, ts->p_id );
 103     check( output, OF_JUNIT, ts->p_id, log_successful_tests );
 104     check( output, OF_JUNIT, ts->p_id, log_cpp_exception_errors );
 105 }
 106
 107 struct guard {
 108     ~guard()
 109     {
 110         boost::unit_test::unit_test_log.set_format( runtime_config::get<output_format>( runtime_config::btrt_log_format ) );
 111         boost::unit_test::unit_test_log.set_stream( std::cout );
 112     }
 113 };
 114
 115
 116 BOOST_AUTO_TEST_CASE( test_logs )
 117 {
 118     guard G;
 119     boost::ignore_unused( G );
 120
 121 #define PATTERN_FILE_NAME "log-count-skipped-tests.pattern"
 122
 123     std::string pattern_file_name(
 124         framework::master_test_suite().argc <= 1
 125             ? (runtime_config::save_pattern() ? PATTERN_FILE_NAME : "./baseline-outputs/" PATTERN_FILE_NAME )
 126             : framework::master_test_suite().argv[1] );
 127
 128     output_test_stream_for_loggers test_output( pattern_file_name,
 129                                                 !runtime_config::save_pattern(),
 130                                                 true,
 131                                                 __FILE__ );
 132
 133     test_case* tc1 = BOOST_TEST_CASE(test_1);
 134     test_case* tc2 = BOOST_TEST_CASE(test_2);
 135     test_case* tc3 = BOOST_TEST_CASE(test_3);
 136
 137     // add decorators to the tests, should happen only once. The status will be reset in the check.
 138     decorator::collector_t* decorator_collector = &(*utf::precondition(skip_with_message("-some precondition-")));
 139     decorator_collector->store_in( *tc2 );
 140     decorator_collector->reset();
 141
 142     decorator_collector = &(* utf::disabled());
 143     decorator_collector->store_in( *tc3 );
 144     decorator_collector->reset();
 145
 146     test_suite* ts_main = BOOST_TEST_SUITE( "fake master test suite" );
 147         ts_main->add( tc1 );
 148         ts_main->add( tc2 );
 149         ts_main->add( tc3 );
 150
 151     check( test_output, ts_main );
 152
 153     // change precondition
 154     skip_with_message::default_enabled = true;
 155     check( test_output, ts_main );
 156
 157     //
 158     // disabling sub suites and subtests
 159     test_suite* ts_main2 = BOOST_TEST_SUITE( "fake master test suite2" );
 160         ts_main2->add( tc1 ); // active
 161         ts_main2->add( tc2 ); // conditionally disabled
 162         ts_main2->add( tc3 ); // disabled
 163
 164     // all disabled: count increases by 2
 165     test_suite* ts_sub1 = BOOST_TEST_SUITE( "child1" );
 166         ts_sub1->add( BOOST_TEST_CASE_NAME(test_1, "t1"));
 167         ts_sub1->add( BOOST_TEST_CASE_NAME(test_1, "t2"));
 168
 169     test_case* tc_2_1 = BOOST_TEST_CASE(test_1); // disabled
 170     test_suite* ts_sub2 = BOOST_TEST_SUITE( "child2" ); // conditionally disabled
 171         ts_sub2->add( tc_2_1 );
 172         ts_sub2->add( BOOST_TEST_CASE_NAME(test_1, "t2"));
 173
 174     ts_main2->add(ts_sub1);
 175     ts_main2->add(ts_sub2);
 176
 177
 178     decorator_collector = &(* utf::disabled());
 179     decorator_collector->store_in( *ts_sub1 );
 180     decorator_collector->reset();
 181
 182     decorator_collector = &(* utf::disabled());
 183     decorator_collector->store_in( *tc_2_1 );
 184     decorator_collector->reset();
 185
 186     decorator_collector = &(*utf::precondition(skip_with_message("-some precondition-")));
 187     decorator_collector->store_in( *ts_sub2 );
 188     decorator_collector->reset();
 189
 190
 191     skip_with_message::default_enabled = false;
 192     check( test_output, ts_main2 );
 193     // count disabled = 2 (main) + 2 (ts_sub1) + 2 (ts_sub2)
 194
 195     // change precondition
 196     skip_with_message::default_enabled = true;
 197     check( test_output, ts_main2 );
 198     // count disabled = 1 (main) + 2 (ts_sub1) + 1 (ts_sub2)
 199 }