1 // (C) Copyright Raffi Enficiaud 2016.
2 // Distributed under the Boost Software License, Version 1.0.
3 // (See accompanying file LICENSE_1_0.txt or copy at
4 // http://www.boost.org/LICENSE_1_0.txt)
5
6 // See http://www.boost.org/libs/test for the library home page.
7 //
8 /// @file
9 /// @brief tests order of the running unit tests under shuffling
10 // ***************************************************************************
11
12 // Boost.Test
13 #define BOOST_TEST_MODULE test unit order shuffled test
14 #include <boost/test/included/unit_test.hpp>
15 #include <boost/test/unit_test_log.hpp>
16 #include <boost/test/tree/visitor.hpp>
17 #include <boost/test/utils/string_cast.hpp>
18
19 #include <boost/test/utils/nullstream.hpp>
20 typedef boost::onullstream onullstream_type;
21 namespace ut = boost::unit_test;
22 namespace tt = boost::test_tools;
23
24 #include <cstddef>
25 #include <iostream>
26
27 //____________________________________________________________________________//
28
some_test()29 void some_test() {}
30 #define TC( name ) \
31 boost::unit_test::make_test_case( boost::function<void ()>(some_test), \
32 BOOST_TEST_STRINGIZE( name ), \
33 __FILE__, __LINE__ )
34
35 //____________________________________________________________________________//
36
37 struct tu_order_collector : ut::test_observer {
test_unit_starttu_order_collector38 virtual void test_unit_start( ut::test_unit const& tu )
39 {
40 //std::cout << "## TU: " << tu.full_name() << std::endl;
41 m_order.push_back( tu.p_id );
42 }
43
44 std::vector<ut::test_unit_id> m_order;
45 };
46
47 //____________________________________________________________________________//
48
49 static tu_order_collector
run_tree(ut::test_suite * master)50 run_tree( ut::test_suite* master )
51 {
52 std::cout << "## TU: START" << std::endl;
53 tu_order_collector c;
54 ut::framework::register_observer( c );
55
56 master->p_default_status.value = ut::test_unit::RS_ENABLED;
57 ut::framework::finalize_setup_phase( master->p_id );
58 ut::framework::impl::setup_for_execution( *master );
59
60 onullstream_type null_output;
61 ut::unit_test_log.set_stream( null_output );
62
63 ut::framework::run( master );
64 ut::unit_test_log.set_stream( std::cout );
65
66 ut::framework::deregister_observer( c );
67
68 return c;
69 }
70
71 //____________________________________________________________________________//
72
73 struct test_tree {
test_treetest_tree74 test_tree() {
75 master = BOOST_TEST_SUITE( "master" );
76
77 std::size_t nb_ts = (std::max)(3, std::rand() % 17);
78 std::vector<ut::test_suite*> tsuites(1, master); // master is in there
79
80 for(std::size_t s = 0; s < nb_ts; s++)
81 {
82 tsuites.push_back(BOOST_TEST_SUITE( "ts1_" + boost::unit_test::utils::string_cast(s)));
83 master->add( tsuites.back() );
84 }
85
86 std::size_t nb_ts2 = (std::max)(3, std::rand() % 11);
87 for(std::size_t s = 0; s < nb_ts2; s++)
88 {
89 tsuites.push_back(BOOST_TEST_SUITE( "ts2_" + boost::unit_test::utils::string_cast(s)));
90 tsuites[std::rand() % nb_ts]->add( tsuites.back() ); // picking a random one in the first level
91 }
92
93 // generating N tests units, associating them to an aribtrary test suite
94 for(std::size_t s = 0; s < 10; s++)
95 {
96 ut::test_case* tc = boost::unit_test::make_test_case(
97 boost::function<void ()>(some_test),
98 "tc_" + boost::unit_test::utils::string_cast(s),
99 __FILE__, __LINE__ );
100 tsuites[std::rand() % tsuites.size()]->add(tc);
101 }
102
103 }
104
105 ut::test_suite* master;
106 };
107
108 //____________________________________________________________________________//
BOOST_FIXTURE_TEST_CASE(test_no_seed,test_tree)109 BOOST_FIXTURE_TEST_CASE( test_no_seed, test_tree )
110 {
111 // no seed set
112 ut::runtime_config::s_arguments_store.set<unsigned int>(ut::runtime_config::btrt_random_seed, 0);
113
114 tu_order_collector res1 = run_tree( master );
115 tu_order_collector res2 = run_tree( master );
116
117 BOOST_TEST( res1.m_order == res2.m_order, tt::per_element() );
118 }
119
BOOST_FIXTURE_TEST_CASE(test_seed_to_time,test_tree)120 BOOST_FIXTURE_TEST_CASE( test_seed_to_time, test_tree )
121 {
122 // seed = 1 means current time is used.
123 ut::runtime_config::s_arguments_store.set<unsigned int>(ut::runtime_config::btrt_random_seed, 1);
124
125 tu_order_collector res1 = run_tree( master );
126 tu_order_collector res2 = run_tree( master );
127
128 BOOST_TEST( res1.m_order != res2.m_order ); // some elements might be the same, but not the full sequences
129 }
130
BOOST_FIXTURE_TEST_CASE(test_seed_identical,test_tree)131 BOOST_FIXTURE_TEST_CASE( test_seed_identical, test_tree )
132 {
133 // seed = 1 means current time is used.
134 unsigned int seed = static_cast<unsigned int>( std::time( 0 ) );
135 ut::runtime_config::s_arguments_store.set<unsigned int>(ut::runtime_config::btrt_random_seed, seed);
136 tu_order_collector res1 = run_tree( master );
137
138 ut::runtime_config::s_arguments_store.set<unsigned int>(ut::runtime_config::btrt_random_seed, seed);
139 tu_order_collector res2 = run_tree( master );
140
141 BOOST_TEST( res1.m_order == res2.m_order, tt::per_element() );
142
143 // using time seed now
144 ut::runtime_config::s_arguments_store.set<unsigned int>(ut::runtime_config::btrt_random_seed, 1);
145 tu_order_collector res3 = run_tree( master );
146 BOOST_TEST( res1.m_order != res3.m_order ); // some elements might be the same, but not the full sequences
147
148
149 }
150