1 // (C) Copyright Raffi Enficiaud 2019.
2 // Distributed under the Boost Software License, Version 1.0.
3 // (See accompanying file LICENSE_1_0.txt or copy at
4 // http://www.boost.org/LICENSE_1_0.txt)
5
6 // See http://www.boost.org/libs/test for the library home page.
7 //
8 // ***************************************************************************
9
10 // Boost.Test
11
12 #define BOOST_TEST_MODULE junit count skipped tests
13 #include <boost/test/included/unit_test.hpp>
14
15 // our special logger for tests
16 #include "logger-for-tests.hpp"
17
18 // STL
19 #include <iostream>
20 #include <ios>
21
22 using boost::test_tools::output_test_stream;
23 using namespace boost::unit_test;
24 namespace utf = boost::unit_test;
25 namespace tt = boost::test_tools;
26
27
28 struct skip_with_message
29 {
30 static bool default_enabled; // to control the status from outside
31
32 std::string message;
skip_with_messageskip_with_message33 skip_with_message(std::string m)
34 : message(m) {}
35
operator ()skip_with_message36 tt::assertion_result operator()(utf::test_unit_id)
37 {
38 tt::assertion_result ans(skip_with_message::default_enabled);
39 ans.message() << "test is skipped because " << message;
40 return ans;
41 }
42 };
43 bool skip_with_message::default_enabled = false;
44
test_1()45 void test_1()
46 {
47 BOOST_CHECK_EQUAL( 2 + 2, 4 );
48 }
49
test_2()50 void test_2()
51 {
52 BOOST_CHECK_EQUAL( 0, 1 );
53 }
54
test_3()55 void test_3()
56 {
57 BOOST_CHECK_EQUAL( 0, 1 );
58 }
59
check(output_test_stream & output,output_format log_format,test_unit_id id,log_level ll=log_successful_tests)60 void check( output_test_stream& output,
61 output_format log_format,
62 test_unit_id id,
63 log_level ll = log_successful_tests )
64 {
65 class reset_status : public test_tree_visitor {
66 private:
67 virtual bool visit( test_unit const& tu )
68 {
69 const_cast<test_unit&>(tu).p_default_status.value = test_case::RS_INHERIT;
70 const_cast<test_unit&>(tu).p_run_status.value = test_case::RS_INVALID;
71 return true;
72 }
73 } rstatus;
74
75 {
76 log_setup_teardown holder(output, log_format, ll);
77
78 // reinit the default/run status otherwise we cannot apply the decorators
79 // after the first run
80 traverse_test_tree( id, rstatus, true );
81 framework::get<test_suite>(id).p_default_status.value = test_unit::RS_ENABLED;
82
83 output << "* " << log_format << "-format *******************************************************************";
84 output << std::endl;
85
86 framework::finalize_setup_phase( id );
87 framework::run( id, false ); // do not continue the test tree to have the test_log_start/end
88 output << std::endl;
89 }
90
91 BOOST_TEST( output.match_pattern(true) ); // flushes the stream at the end of the comparison.
92 }
93
94 //____________________________________________________________________________//
95
check(output_test_stream & output,test_suite * ts)96 void check( output_test_stream& output, test_suite* ts )
97 {
98 check( output, OF_CLF, ts->p_id );
99 check( output, OF_XML, ts->p_id );
100 check( output, OF_JUNIT, ts->p_id, log_successful_tests );
101 check( output, OF_JUNIT, ts->p_id, log_cpp_exception_errors );
102 }
103
104 struct guard {
~guardguard105 ~guard()
106 {
107 boost::unit_test::unit_test_log.set_format( runtime_config::get<output_format>( runtime_config::btrt_log_format ) );
108 boost::unit_test::unit_test_log.set_stream( std::cout );
109 }
110 };
111
112
BOOST_AUTO_TEST_CASE(test_logs)113 BOOST_AUTO_TEST_CASE( test_logs )
114 {
115 guard G;
116 boost::ignore_unused( G );
117
118 #define PATTERN_FILE_NAME "log-count-skipped-tests.pattern"
119
120 std::string pattern_file_name(
121 framework::master_test_suite().argc <= 1
122 ? (runtime_config::save_pattern() ? PATTERN_FILE_NAME : "./baseline-outputs/" PATTERN_FILE_NAME )
123 : framework::master_test_suite().argv[1] );
124
125 output_test_stream_for_loggers test_output( pattern_file_name,
126 !runtime_config::save_pattern(),
127 true,
128 __FILE__ );
129
130 test_case* tc1 = BOOST_TEST_CASE(test_1);
131 test_case* tc2 = BOOST_TEST_CASE(test_2);
132 test_case* tc3 = BOOST_TEST_CASE(test_3);
133
134 // add decorators to the tests, should happen only once. The status will be reset in the check.
135 decorator::collector_t* decorator_collector = &(*utf::precondition(skip_with_message("-some precondition-")));
136 decorator_collector->store_in( *tc2 );
137 decorator_collector->reset();
138
139 decorator_collector = &(* utf::disabled());
140 decorator_collector->store_in( *tc3 );
141 decorator_collector->reset();
142
143 test_suite* ts_main = BOOST_TEST_SUITE( "fake master test suite" );
144 ts_main->add( tc1 );
145 ts_main->add( tc2 );
146 ts_main->add( tc3 );
147
148 check( test_output, ts_main );
149
150 // change precondition
151 skip_with_message::default_enabled = true;
152 check( test_output, ts_main );
153
154 //
155 // disabling sub suites and subtests
156 test_suite* ts_main2 = BOOST_TEST_SUITE( "fake master test suite2" );
157 ts_main2->add( tc1 ); // active
158 ts_main2->add( tc2 ); // conditionally disabled
159 ts_main2->add( tc3 ); // disabled
160
161 // all disabled: count increases by 2
162 test_suite* ts_sub1 = BOOST_TEST_SUITE( "child1" );
163 ts_sub1->add( BOOST_TEST_CASE_NAME(test_1, "t1"));
164 ts_sub1->add( BOOST_TEST_CASE_NAME(test_1, "t2"));
165
166 test_case* tc_2_1 = BOOST_TEST_CASE(test_1); // disabled
167 test_suite* ts_sub2 = BOOST_TEST_SUITE( "child2" ); // conditionally disabled
168 ts_sub2->add( tc_2_1 );
169 ts_sub2->add( BOOST_TEST_CASE_NAME(test_1, "t2"));
170
171 ts_main2->add(ts_sub1);
172 ts_main2->add(ts_sub2);
173
174
175 decorator_collector = &(* utf::disabled());
176 decorator_collector->store_in( *ts_sub1 );
177 decorator_collector->reset();
178
179 decorator_collector = &(* utf::disabled());
180 decorator_collector->store_in( *tc_2_1 );
181 decorator_collector->reset();
182
183 decorator_collector = &(*utf::precondition(skip_with_message("-some precondition-")));
184 decorator_collector->store_in( *ts_sub2 );
185 decorator_collector->reset();
186
187
188 skip_with_message::default_enabled = false;
189 check( test_output, ts_main2 );
190 // count disabled = 2 (main) + 2 (ts_sub1) + 2 (ts_sub2)
191
192 // change precondition
193 skip_with_message::default_enabled = true;
194 check( test_output, ts_main2 );
195 // count disabled = 1 (main) + 2 (ts_sub1) + 1 (ts_sub2)
196 }
197