1 // (C) Copyright Raffi Enficiaud 2016.
2 // Distributed under the Boost Software License, Version 1.0.
3 // (See accompanying file LICENSE_1_0.txt or copy at
4 // http://www.boost.org/LICENSE_1_0.txt)
5
6 // See http://www.boost.org/libs/test for the library home page.
7 //
8 // tests Unit Test Framework logging facilities against
9 // pattern file
10 // ***************************************************************************
11
12 // Boost.Test
13 #define BOOST_TEST_MAIN
14 #include <boost/test/unit_test.hpp>
15 #include <boost/test/unit_test_log.hpp>
16 #include <boost/test/unit_test_suite.hpp>
17 #include <boost/test/framework.hpp>
18 #include <boost/test/unit_test_parameters.hpp>
19 #include <boost/test/utils/nullstream.hpp>
20 #include <boost/test/execution_monitor.hpp>
21
22 typedef boost::onullstream onullstream_type;
23
24 // BOOST
25 #include <boost/lexical_cast.hpp>
26
27 // our special logger for tests
28 #include "logger-for-tests.hpp"
29
30 // STL
31 #include <iostream>
32 #include <ios>
33
34 using boost::test_tools::output_test_stream;
35 using namespace boost::unit_test;
36
37
38
39 //____________________________________________________________________________//
40
41 // this one should generate a message as it does not execute any assertion
good_foo()42 void good_foo() {}
43
almost_good_foo()44 void almost_good_foo()
45 {
46 BOOST_TEST_WARN( 2>3 );
47 }
48
bad_foo()49 void bad_foo() {
50 BOOST_ERROR( "" );
51
52 BOOST_TEST_MESSAGE("this is a message");
53 BOOST_CHECK(true);
54
55 BOOST_TEST_INFO("Context value=something");
56 BOOST_TEST_INFO("Context value2=something different");
57 BOOST_ERROR( "with some message" );
58
59 BOOST_CHECK_MESSAGE( 1 == 2.3, "non sense" );
60 }
61
very_bad_foo()62 void very_bad_foo() {
63 BOOST_TEST_CONTEXT("some context") {
64 BOOST_FAIL( "very_bad_foo is fatal" );
65 }
66 }
67
68 struct local_exception {};
69
very_bad_exception()70 void very_bad_exception() {
71 BOOST_TEST_INFO("Context value=something");
72 BOOST_TEST_INFO("Context value2=something different");
73 BOOST_ERROR( "with some message" );
74
75 BOOST_TEST_INFO("exception context should be shown");
76 throw local_exception();
77 }
78
bad_foo2()79 void bad_foo2() { bad_foo(); } // tests with clashing names
80
timeout_foo()81 void timeout_foo()
82 {
83 using boost::execution_exception;
84 execution_exception::location dummy;
85 throw execution_exception(
86 execution_exception::timeout_error,
87 "fake timeout",
88 dummy);
89 }
90
context_mixed_foo()91 void context_mixed_foo() {
92 BOOST_TEST_CONTEXT("some context") {
93 BOOST_TEST( true );
94 }
95
96 BOOST_TEST_INFO("other context");
97 throw local_exception();
98 }
99
100 //____________________________________________________________________________//
101
check(output_test_stream & output,output_format log_format,test_unit_id id,log_level ll=log_successful_tests,output_format additional_format=OF_INVALID)102 void check( output_test_stream& output,
103 output_format log_format,
104 test_unit_id id,
105 log_level ll = log_successful_tests,
106 output_format additional_format = OF_INVALID)
107 {
108 {
109 log_setup_teardown holder(output, log_format, ll, invalid_log_level, additional_format);
110 output << "* " << log_format << "-format *******************************************************************";
111 output << std::endl;
112 framework::finalize_setup_phase( id );
113 framework::run( id, false ); // do not continue the test tree to have the test_log_start/end
114 output << std::endl;
115 }
116
117 BOOST_TEST( output.match_pattern(true) ); // flushes the stream at the end of the comparison.
118 }
119
120 //____________________________________________________________________________//
121
check(output_test_stream & output,test_suite * ts)122 void check( output_test_stream& output, test_suite* ts )
123 {
124 ts->p_default_status.value = test_unit::RS_ENABLED;
125
126 check( output, OF_CLF, ts->p_id );
127 check( output, OF_XML, ts->p_id );
128 check( output, OF_JUNIT, ts->p_id, log_successful_tests );
129 check( output, OF_JUNIT, ts->p_id, log_cpp_exception_errors ); // should branch to the log log_all_errors
130 }
131
132 //____________________________________________________________________________//
133
134 struct guard {
~guardguard135 ~guard()
136 {
137 boost::unit_test::unit_test_log.set_format( runtime_config::get<output_format>( runtime_config::btrt_log_format ) );
138 boost::unit_test::unit_test_log.set_stream( std::cout );
139 }
140 };
141
142 //____________________________________________________________________________//
143
144 struct save_arguments {
save_patternsave_arguments145 static bool save_pattern() {
146 if(m_first) {
147 m_save_pattern = runtime_config::save_pattern();
148 m_first = false;
149 }
150 return m_save_pattern;
151 }
152
saved_argvsave_arguments153 static const std::vector<std::string>& saved_argv() {
154 if(argv.empty()) {
155 for(size_t i = 0; i < static_cast<size_t>(framework::master_test_suite().argc); i++) {
156 argv.push_back(framework::master_test_suite().argv[i]);
157 }
158 }
159 return argv;
160 }
161
get_pattern_filesave_arguments162 static std::string get_pattern_file(const std::string &to_retrieve) {
163 if(argv.empty()) {
164 for(size_t i = 0; i < static_cast<size_t>(framework::master_test_suite().argc); i++) {
165 argv.push_back(framework::master_test_suite().argv[i]);
166 }
167 }
168
169 for(size_t i = 0; i < static_cast<size_t>(framework::master_test_suite().argc); i++) {
170 if(argv[i].find(to_retrieve) != std::string::npos)
171 return argv[i];
172 }
173
174 return "";
175 }
176
177 static std::vector<std::string> argv;
178 static bool m_first;
179 static bool m_save_pattern;
180 };
181
182 bool save_arguments::m_save_pattern = false;
183 bool save_arguments::m_first = true;
184 std::vector<std::string> save_arguments::argv = std::vector<std::string>();
185
186 //____________________________________________________________________________//
187
188
BOOST_AUTO_TEST_CASE(test_logs)189 BOOST_AUTO_TEST_CASE( test_logs )
190 {
191 guard G;
192 boost::ignore_unused( G );
193
194 #define PATTERN_FILE_NAME "log-formatter-test.pattern"
195
196 std::string pattern_file_name(
197 // we cannot use runtime_config::save_arguments() as one of the test is mutating
198 // the arguments for testing purposes
199 // we have to inspect argv as b2 may run the program from an unknown location
200 save_arguments::saved_argv().size() <= 1
201 ? (save_arguments::save_pattern()
202 ? PATTERN_FILE_NAME
203 : "./baseline-outputs/" PATTERN_FILE_NAME )
204 : save_arguments::get_pattern_file(PATTERN_FILE_NAME));
205
206 output_test_stream_for_loggers test_output( pattern_file_name,
207 !save_arguments::save_pattern(),
208 true,
209 __FILE__ );
210
211 #line 157
212 test_suite* ts_0 = BOOST_TEST_SUITE( "0 test cases inside" );
213
214 test_suite* ts_1 = BOOST_TEST_SUITE( "1 test cases inside" );
215 ts_1->add( BOOST_TEST_CASE( good_foo ) );
216
217 test_suite* ts_1b = BOOST_TEST_SUITE( "1 bad test case inside" );
218 ts_1b->add( BOOST_TEST_CASE( bad_foo ), 1 );
219
220 test_suite* ts_1c = BOOST_TEST_SUITE( "1 almost good test case inside" );
221 ts_1c->add( BOOST_TEST_CASE( almost_good_foo ) );
222
223 test_suite* ts_2 = BOOST_TEST_SUITE( "2 test cases inside" );
224 ts_2->add( BOOST_TEST_CASE( good_foo ) );
225 ts_2->add( BOOST_TEST_CASE( bad_foo ), 1 );
226
227 test_suite* ts_3 = BOOST_TEST_SUITE( "3 test cases inside" );
228 ts_3->add( BOOST_TEST_CASE( bad_foo ) );
229 test_case* tc1 = BOOST_TEST_CASE( very_bad_foo );
230 ts_3->add( tc1 );
231 test_case* tc2 = BOOST_TEST_CASE_NAME( bad_foo2 , "bad_foo2<int>" ); // this is for skipped message GH-253
232 ts_3->add( tc2 );
233 tc2->depends_on( tc1 );
234
235 test_suite* ts_4 = BOOST_TEST_SUITE( "4 test cases inside" );
236 ts_4->add( BOOST_TEST_CASE( bad_foo ) );
237 ts_4->add( BOOST_TEST_CASE( very_bad_foo ) );
238 ts_4->add( BOOST_TEST_CASE( very_bad_exception ) );
239 ts_4->add( BOOST_TEST_CASE( bad_foo2 ) );
240
241 test_suite* ts_main = BOOST_TEST_SUITE( "Fake Test Suite Hierarchy" );
242 ts_main->add( ts_0 );
243 ts_main->add( ts_1 );
244 ts_main->add( ts_2 );
245 ts_main->add( ts_3 );
246 ts_main->add( ts_4 );
247
248 test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" );
249 ts_timeout->add( BOOST_TEST_CASE( good_foo ) );
250 test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo );
251 ts_timeout->add( tc_timeout );
252
253 test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" );
254 ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) );
255 test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" );
256 ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) );
257 test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo );
258 ts_timeout_internal->add( tc_timeout_internal );
259 ts_timeout_nested->add( ts_timeout_internal );
260 ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) );
261
262 check( test_output, ts_1 );
263
264 check( test_output, ts_1b );
265
266 check( test_output, ts_1c );
267
268 check( test_output, ts_2 );
269
270 check( test_output, ts_3 );
271
272 check( test_output, ts_4 );
273
274 ts_1->add( BOOST_TEST_CASE_NAME( bad_foo, "bad<bool>" ) );
275 ts_3->depends_on( ts_1 );
276
277 check( test_output, ts_main );
278
279 check( test_output, ts_timeout );
280
281 check( test_output, ts_timeout_nested );
282 }
283
284
BOOST_AUTO_TEST_CASE(test_logs_junit_info_closing_tags)285 BOOST_AUTO_TEST_CASE( test_logs_junit_info_closing_tags )
286 {
287 guard G;
288 boost::ignore_unused( G );
289
290 #define PATTERN_FILE_NAME_JUNIT "log-formatter-test.pattern.junit"
291
292 std::string pattern_file_name(
293 // we cannot use runtime_config::save_arguments() as one of the test is mutating
294 // the arguments for testing purposes
295 // we have to inspect argv as b2 may run the program from an unknown location
296 save_arguments::saved_argv().size() <= 1
297 ? (save_arguments::save_pattern()
298 ? PATTERN_FILE_NAME_JUNIT
299 : "./baseline-outputs/" PATTERN_FILE_NAME_JUNIT )
300 : save_arguments::get_pattern_file(PATTERN_FILE_NAME_JUNIT));
301
302 output_test_stream_for_loggers test_output( pattern_file_name,
303 !save_arguments::save_pattern(),
304 true,
305 __FILE__ );
306
307 #line 249
308 test_suite* ts_main = BOOST_TEST_SUITE( "1 test cases inside" );
309 ts_main->add( BOOST_TEST_CASE( almost_good_foo ) );
310
311 ts_main->p_default_status.value = test_unit::RS_ENABLED;
312 char const* argv[] = { "a.exe", "--run_test=*", "--build_info" };
313 int argc = sizeof(argv)/sizeof(argv[0]);
314 boost::unit_test::runtime_config::init( argc, (char**)argv );
315 boost::unit_test::framework::impl::setup_for_execution( *ts_main );
316
317 check( test_output, OF_JUNIT, ts_main->p_id, log_successful_tests );
318
319 test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" );
320 ts_timeout->add( BOOST_TEST_CASE( good_foo ) );
321 test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo );
322 ts_timeout->add( tc_timeout );
323
324 check( test_output, OF_JUNIT, ts_timeout->p_id, log_successful_tests );
325
326
327 test_suite* ts_account_failures = BOOST_TEST_SUITE( "1 junit failure is not error" );
328 ts_account_failures->add( BOOST_TEST_CASE( bad_foo ) );
329 ts_account_failures->add( BOOST_TEST_CASE( very_bad_foo ) );
330 ts_account_failures->add( BOOST_TEST_CASE( good_foo ) );
331 ts_account_failures->add( BOOST_TEST_CASE( bad_foo2 ) );
332
333 char const* argv2[] = { "a.exe", "--run_test=*", "--build_info=false" };
334 int argc2 = sizeof(argv2)/sizeof(argv2[0]);
335 boost::unit_test::runtime_config::init( argc2, (char**)argv2 );
336 boost::unit_test::framework::impl::setup_for_execution( *ts_account_failures );
337
338 check( test_output, OF_JUNIT, ts_account_failures->p_id, log_messages );
339 }
340
341
BOOST_AUTO_TEST_CASE(test_logs_context)342 BOOST_AUTO_TEST_CASE( test_logs_context )
343 {
344 guard G;
345 boost::ignore_unused( G );
346
347 #define PATTERN_FILE_NAME_CONTEXT "log-formatter-context-test.pattern"
348
349 std::string pattern_file_name(
350 // we cannot use runtime_config::save_arguments() as one of the test is mutating
351 // the arguments for testing purposes
352 // we have to inspect argv as b2 may run the program from an unknown location
353 save_arguments::saved_argv().size() <= 1
354 ? (save_arguments::save_pattern()
355 ? PATTERN_FILE_NAME_CONTEXT
356 : "./baseline-outputs/" PATTERN_FILE_NAME_CONTEXT )
357 : save_arguments::get_pattern_file(PATTERN_FILE_NAME_CONTEXT));
358
359 output_test_stream_for_loggers test_output( pattern_file_name,
360 !save_arguments::save_pattern(),
361 true,
362 __FILE__ );
363
364 #line 330
365 test_suite* ts_1 = BOOST_TEST_SUITE( "1 test cases inside" );
366 ts_1->add( BOOST_TEST_CASE( context_mixed_foo ) );
367
368 ts_1->p_default_status.value = test_unit::RS_ENABLED;
369 check( test_output, OF_CLF, ts_1->p_id, log_successful_tests );
370 check( test_output, OF_CLF, ts_1->p_id, log_cpp_exception_errors );
371 check( test_output, OF_CLF, ts_1->p_id, log_successful_tests, OF_JUNIT );
372 check( test_output, OF_CLF, ts_1->p_id, log_cpp_exception_errors, OF_JUNIT );
373
374 check( test_output, OF_XML, ts_1->p_id, log_successful_tests );
375 check( test_output, OF_XML, ts_1->p_id, log_cpp_exception_errors );
376 check( test_output, OF_JUNIT, ts_1->p_id, log_successful_tests );
377 check( test_output, OF_JUNIT, ts_1->p_id, log_cpp_exception_errors );
378
379 }
380
381
382 // EOF
383