1 /*
2 * Created by Phil on 13/5/2013.
3 * Copyright 2014 Two Blue Cubes Ltd. All rights reserved.
4 *
5 * Distributed under the Boost Software License, Version 1.0. (See accompanying
6 * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 */
8
9 #include "catch.hpp"
10 #include "internal/catch_test_spec_parser.h"
11 #include "internal/catch_config.hpp"
12 #include "internal/catch_commandline.h"
13
14 #ifdef __clang__
15 # pragma clang diagnostic ignored "-Wc++98-compat"
16 #endif
17
fakeTestCase(const char * name,const char * desc="")18 inline Catch::TestCase fakeTestCase(const char* name, const char* desc = "") { return Catch::makeTestCase(nullptr, "", { name, desc }, CATCH_INTERNAL_LINEINFO); }
19
20 TEST_CASE( "Parse test names and tags" ) {
21
22 using Catch::parseTestSpec;
23 using Catch::TestSpec;
24
25 Catch::TestCase tcA = fakeTestCase( "a" );
26 Catch::TestCase tcB = fakeTestCase( "b", "[one][x]" );
27 Catch::TestCase tcC = fakeTestCase( "longer name with spaces", "[two][three][.][x]" );
28 Catch::TestCase tcD = fakeTestCase( "zlonger name with spacesz" );
29
30 SECTION( "Empty test spec should have no filters" ) {
31 TestSpec spec;
32 CHECK( spec.hasFilters() == false );
33 CHECK( spec.matches( tcA ) == false );
34 CHECK( spec.matches( tcB ) == false );
35 }
36
37 SECTION( "Test spec from empty string should have no filters" ) {
38 TestSpec spec = parseTestSpec( "" );
39 CHECK( spec.hasFilters() == false );
40 CHECK( spec.matches(tcA ) == false );
41 CHECK( spec.matches( tcB ) == false );
42 }
43
44 SECTION( "Test spec from just a comma should have no filters" ) {
45 TestSpec spec = parseTestSpec( "," );
46 CHECK( spec.hasFilters() == false );
47 CHECK( spec.matches( tcA ) == false );
48 CHECK( spec.matches( tcB ) == false );
49 }
50
51 SECTION( "Test spec from name should have one filter" ) {
52 TestSpec spec = parseTestSpec( "b" );
53 CHECK( spec.hasFilters() == true );
54 CHECK( spec.matches( tcA ) == false );
55 CHECK( spec.matches( tcB ) == true );
56 }
57
58 SECTION( "Test spec from quoted name should have one filter" ) {
59 TestSpec spec = parseTestSpec( "\"b\"" );
60 CHECK( spec.hasFilters() == true );
61 CHECK( spec.matches( tcA ) == false );
62 CHECK( spec.matches( tcB ) == true );
63 }
64
65 SECTION( "Test spec from name should have one filter" ) {
66 TestSpec spec = parseTestSpec( "b" );
67 CHECK( spec.hasFilters() == true );
68 CHECK( spec.matches( tcA ) == false );
69 CHECK( spec.matches( tcB ) == true );
70 CHECK( spec.matches( tcC ) == false );
71 }
72
73 SECTION( "Wildcard at the start" ) {
74 TestSpec spec = parseTestSpec( "*spaces" );
75 CHECK( spec.hasFilters() == true );
76 CHECK( spec.matches( tcA ) == false );
77 CHECK( spec.matches( tcB ) == false );
78 CHECK( spec.matches( tcC ) == true );
79 CHECK( spec.matches( tcD ) == false );
80 CHECK( parseTestSpec( "*a" ).matches( tcA ) == true );
81 }
82 SECTION( "Wildcard at the end" ) {
83 TestSpec spec = parseTestSpec( "long*" );
84 CHECK( spec.hasFilters() == true );
85 CHECK( spec.matches( tcA ) == false );
86 CHECK( spec.matches( tcB ) == false );
87 CHECK( spec.matches( tcC ) == true );
88 CHECK( spec.matches( tcD ) == false );
89 CHECK( parseTestSpec( "a*" ).matches( tcA ) == true );
90 }
91 SECTION( "Wildcard at both ends" ) {
92 TestSpec spec = parseTestSpec( "*name*" );
93 CHECK( spec.hasFilters() == true );
94 CHECK( spec.matches( tcA ) == false );
95 CHECK( spec.matches( tcB ) == false );
96 CHECK( spec.matches( tcC ) == true );
97 CHECK( spec.matches( tcD ) == true );
98 CHECK( parseTestSpec( "*a*" ).matches( tcA ) == true );
99 }
100 SECTION( "Redundant wildcard at the start" ) {
101 TestSpec spec = parseTestSpec( "*a" );
102 CHECK( spec.hasFilters() == true );
103 CHECK( spec.matches( tcA ) == true );
104 CHECK( spec.matches( tcB ) == false );
105 }
106 SECTION( "Redundant wildcard at the end" ) {
107 TestSpec spec = parseTestSpec( "a*" );
108 CHECK( spec.hasFilters() == true );
109 CHECK( spec.matches( tcA ) == true );
110 CHECK( spec.matches( tcB ) == false );
111 }
112 SECTION( "Redundant wildcard at both ends" ) {
113 TestSpec spec = parseTestSpec( "*a*" );
114 CHECK( spec.hasFilters() == true );
115 CHECK( spec.matches( tcA ) == true );
116 CHECK( spec.matches( tcB ) == false );
117 }
118 SECTION( "Wildcard at both ends, redundant at start" ) {
119 TestSpec spec = parseTestSpec( "*longer*" );
120 CHECK( spec.hasFilters() == true );
121 CHECK( spec.matches( tcA ) == false );
122 CHECK( spec.matches( tcB ) == false );
123 CHECK( spec.matches( tcC ) == true );
124 CHECK( spec.matches( tcD ) == true );
125 }
126 SECTION( "Just wildcard" ) {
127 TestSpec spec = parseTestSpec( "*" );
128 CHECK( spec.hasFilters() == true );
129 CHECK( spec.matches( tcA ) == true );
130 CHECK( spec.matches( tcB ) == true );
131 CHECK( spec.matches( tcC ) == true );
132 CHECK( spec.matches( tcD ) == true );
133 }
134
135 SECTION( "Single tag" ) {
136 TestSpec spec = parseTestSpec( "[one]" );
137 CHECK( spec.hasFilters() == true );
138 CHECK( spec.matches( tcA ) == false );
139 CHECK( spec.matches( tcB ) == true );
140 CHECK( spec.matches( tcC ) == false );
141 }
142 SECTION( "Single tag, two matches" ) {
143 TestSpec spec = parseTestSpec( "[x]" );
144 CHECK( spec.hasFilters() == true );
145 CHECK( spec.matches( tcA ) == false );
146 CHECK( spec.matches( tcB ) == true );
147 CHECK( spec.matches( tcC ) == true );
148 }
149 SECTION( "Two tags" ) {
150 TestSpec spec = parseTestSpec( "[two][x]" );
151 CHECK( spec.hasFilters() == true );
152 CHECK( spec.matches( tcA ) == false );
153 CHECK( spec.matches( tcB ) == false );
154 CHECK( spec.matches( tcC ) == true );
155 }
156 SECTION( "Two tags, spare separated" ) {
157 TestSpec spec = parseTestSpec( "[two] [x]" );
158 CHECK( spec.hasFilters() == true );
159 CHECK( spec.matches( tcA ) == false );
160 CHECK( spec.matches( tcB ) == false );
161 CHECK( spec.matches( tcC ) == true );
162 }
163 SECTION( "Wildcarded name and tag" ) {
164 TestSpec spec = parseTestSpec( "*name*[x]" );
165 CHECK( spec.hasFilters() == true );
166 CHECK( spec.matches( tcA ) == false );
167 CHECK( spec.matches( tcB ) == false );
168 CHECK( spec.matches( tcC ) == true );
169 CHECK( spec.matches( tcD ) == false );
170 }
171 SECTION( "Single tag exclusion" ) {
172 TestSpec spec = parseTestSpec( "~[one]" );
173 CHECK( spec.hasFilters() == true );
174 CHECK( spec.matches( tcA ) == true );
175 CHECK( spec.matches( tcB ) == false );
176 CHECK( spec.matches( tcC ) == true );
177 }
178 SECTION( "One tag exclusion and one tag inclusion" ) {
179 TestSpec spec = parseTestSpec( "~[two][x]" );
180 CHECK( spec.hasFilters() == true );
181 CHECK( spec.matches( tcA ) == false );
182 CHECK( spec.matches( tcB ) == true );
183 CHECK( spec.matches( tcC ) == false );
184 }
185 SECTION( "One tag exclusion and one wldcarded name inclusion" ) {
186 TestSpec spec = parseTestSpec( "~[two]*name*" );
187 CHECK( spec.hasFilters() == true );
188 CHECK( spec.matches( tcA ) == false );
189 CHECK( spec.matches( tcB ) == false );
190 CHECK( spec.matches( tcC ) == false );
191 CHECK( spec.matches( tcD ) == true );
192 }
193 SECTION( "One tag exclusion, using exclude:, and one wldcarded name inclusion" ) {
194 TestSpec spec = parseTestSpec( "exclude:[two]*name*" );
195 CHECK( spec.hasFilters() == true );
196 CHECK( spec.matches( tcA ) == false );
197 CHECK( spec.matches( tcB ) == false );
198 CHECK( spec.matches( tcC ) == false );
199 CHECK( spec.matches( tcD ) == true );
200 }
201 SECTION( "name exclusion" ) {
202 TestSpec spec = parseTestSpec( "~b" );
203 CHECK( spec.hasFilters() == true );
204 CHECK( spec.matches( tcA ) == true );
205 CHECK( spec.matches( tcB ) == false );
206 CHECK( spec.matches( tcC ) == true );
207 CHECK( spec.matches( tcD ) == true );
208 }
209 SECTION( "wildcarded name exclusion" ) {
210 TestSpec spec = parseTestSpec( "~*name*" );
211 CHECK( spec.hasFilters() == true );
212 CHECK( spec.matches( tcA ) == true );
213 CHECK( spec.matches( tcB ) == true );
214 CHECK( spec.matches( tcC ) == false );
215 CHECK( spec.matches( tcD ) == false );
216 }
217 SECTION( "wildcarded name exclusion with tag inclusion" ) {
218 TestSpec spec = parseTestSpec( "~*name*,[three]" );
219 CHECK( spec.hasFilters() == true );
220 CHECK( spec.matches( tcA ) == true );
221 CHECK( spec.matches( tcB ) == true );
222 CHECK( spec.matches( tcC ) == true );
223 CHECK( spec.matches( tcD ) == false );
224 }
225 SECTION( "wildcarded name exclusion, using exclude:, with tag inclusion" ) {
226 TestSpec spec = parseTestSpec( "exclude:*name*,[three]" );
227 CHECK( spec.hasFilters() == true );
228 CHECK( spec.matches( tcA ) == true );
229 CHECK( spec.matches( tcB ) == true );
230 CHECK( spec.matches( tcC ) == true );
231 CHECK( spec.matches( tcD ) == false );
232 }
233 SECTION( "two wildcarded names" ) {
234 TestSpec spec = parseTestSpec( "\"longer*\"\"*spaces\"" );
235 CHECK( spec.hasFilters() == true );
236 CHECK( spec.matches( tcA ) == false );
237 CHECK( spec.matches( tcB ) == false );
238 CHECK( spec.matches( tcC ) == true );
239 CHECK( spec.matches( tcD ) == false );
240 }
241 SECTION( "empty tag" ) {
242 TestSpec spec = parseTestSpec( "[]" );
243 CHECK( spec.hasFilters() == false );
244 CHECK( spec.matches( tcA ) == false );
245 CHECK( spec.matches( tcB ) == false );
246 CHECK( spec.matches( tcC ) == false );
247 CHECK( spec.matches( tcD ) == false );
248 }
249 SECTION( "empty quoted name" ) {
250 TestSpec spec = parseTestSpec( "\"\"" );
251 CHECK( spec.hasFilters() == false );
252 CHECK( spec.matches( tcA ) == false );
253 CHECK( spec.matches( tcB ) == false );
254 CHECK( spec.matches( tcC ) == false );
255 CHECK( spec.matches( tcD ) == false );
256 }
257 SECTION( "quoted string followed by tag exclusion" ) {
258 TestSpec spec = parseTestSpec( "\"*name*\"~[.]" );
259 CHECK( spec.hasFilters() == true );
260 CHECK( spec.matches( tcA ) == false );
261 CHECK( spec.matches( tcB ) == false );
262 CHECK( spec.matches( tcC ) == false );
263 CHECK( spec.matches( tcD ) == true );
264 }
265
266 }
267
268 TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {
269
270 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
271 using namespace Catch::Matchers;
272 #endif
273
274 Catch::ConfigData config;
275 auto cli = Catch::makeCommandLineParser(config);
276
277 SECTION("empty args don't cause a crash") {
278 auto result = cli.parse({""});
279 CHECK(result);
280 CHECK(config.processName == "");
281 }
282
283 SECTION("default - no arguments") {
284 auto result = cli.parse({"test"});
285 CHECK(result);
286 CHECK(config.processName == "test");
287 CHECK(config.shouldDebugBreak == false);
288 CHECK(config.abortAfter == -1);
289 CHECK(config.noThrow == false);
290 CHECK(config.reporterName == "console");
291
292 Catch::Config cfg(config);
293 CHECK_FALSE(cfg.hasTestFilters());
294 }
295
296 SECTION("test lists") {
297 SECTION("Specify one test case using") {
298 auto result = cli.parse({"test", "test1"});
299 CHECK(result);
300
301 Catch::Config cfg(config);
302 REQUIRE(cfg.hasTestFilters());
303 REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
304 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
305 }
306 SECTION("Specify one test case exclusion using exclude:") {
307 auto result = cli.parse({"test", "exclude:test1"});
308 CHECK(result);
309
310 Catch::Config cfg(config);
311 REQUIRE(cfg.hasTestFilters());
312 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
313 REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
314 }
315
316 SECTION("Specify one test case exclusion using ~") {
317 auto result = cli.parse({"test", "~test1"});
318 CHECK(result);
319
320 Catch::Config cfg(config);
321 REQUIRE(cfg.hasTestFilters());
322 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
323 REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
324 }
325
326 }
327
328 SECTION("reporter") {
329 SECTION("-r/console") {
330 CHECK(cli.parse({"test", "-r", "console"}));
331
332 REQUIRE(config.reporterName == "console");
333 }
334 SECTION("-r/xml") {
335 CHECK(cli.parse({"test", "-r", "xml"}));
336
337 REQUIRE(config.reporterName == "xml");
338 }
339 SECTION("--reporter/junit") {
340 CHECK(cli.parse({"test", "--reporter", "junit"}));
341
342 REQUIRE(config.reporterName == "junit");
343 }
344 SECTION("Only one reporter is accepted") {
345 REQUIRE_FALSE(cli.parse({ "test", "-r", "xml", "-r", "junit" }));
346 }
347 SECTION("must match one of the available ones") {
348 auto result = cli.parse({"test", "--reporter", "unsupported"});
349 CHECK(!result);
350
351 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
352 REQUIRE_THAT(result.errorMessage(), Contains("Unrecognized reporter"));
353 #endif
354 }
355 }
356
357 SECTION("debugger") {
358 SECTION("-b") {
359 CHECK(cli.parse({"test", "-b"}));
360
361 REQUIRE(config.shouldDebugBreak == true);
362 }
363 SECTION("--break") {
364 CHECK(cli.parse({"test", "--break"}));
365
366 REQUIRE(config.shouldDebugBreak);
367 }
368 }
369
370
371 SECTION("abort") {
372 SECTION("-a aborts after first failure") {
373 CHECK(cli.parse({"test", "-a"}));
374
375 REQUIRE(config.abortAfter == 1);
376 }
377 SECTION("-x 2 aborts after two failures") {
378 CHECK(cli.parse({"test", "-x", "2"}));
379
380 REQUIRE(config.abortAfter == 2);
381 }
382 SECTION("-x must be numeric") {
383 auto result = cli.parse({"test", "-x", "oops"});
384 CHECK(!result);
385
386 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
387 REQUIRE_THAT(result.errorMessage(), Contains("convert") && Contains("oops"));
388 #endif
389 }
390 }
391
392 SECTION("nothrow") {
393 SECTION("-e") {
394 CHECK(cli.parse({"test", "-e"}));
395
396 REQUIRE(config.noThrow);
397 }
398 SECTION("--nothrow") {
399 CHECK(cli.parse({"test", "--nothrow"}));
400
401 REQUIRE(config.noThrow);
402 }
403 }
404
405 SECTION("output filename") {
406 SECTION("-o filename") {
407 CHECK(cli.parse({"test", "-o", "filename.ext"}));
408
409 REQUIRE(config.outputFilename == "filename.ext");
410 }
411 SECTION("--out") {
412 CHECK(cli.parse({"test", "--out", "filename.ext"}));
413
414 REQUIRE(config.outputFilename == "filename.ext");
415 }
416 }
417
418 SECTION("combinations") {
419 SECTION("Single character flags can be combined") {
420 CHECK(cli.parse({"test", "-abe"}));
421
422 CHECK(config.abortAfter == 1);
423 CHECK(config.shouldDebugBreak);
424 CHECK(config.noThrow == true);
425 }
426 }
427
428
429 SECTION( "use-colour") {
430
431 using Catch::UseColour;
432
433 SECTION( "without option" ) {
434 CHECK(cli.parse({"test"}));
435
436 REQUIRE( config.useColour == UseColour::Auto );
437 }
438
439 SECTION( "auto" ) {
440 CHECK(cli.parse({"test", "--use-colour", "auto"}));
441
442 REQUIRE( config.useColour == UseColour::Auto );
443 }
444
445 SECTION( "yes" ) {
446 CHECK(cli.parse({"test", "--use-colour", "yes"}));
447
448 REQUIRE( config.useColour == UseColour::Yes );
449 }
450
451 SECTION( "no" ) {
452 CHECK(cli.parse({"test", "--use-colour", "no"}));
453
454 REQUIRE( config.useColour == UseColour::No );
455 }
456
457 SECTION( "error" ) {
458 auto result = cli.parse({"test", "--use-colour", "wrong"});
459 CHECK( !result );
460 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
461 CHECK_THAT( result.errorMessage(), Contains( "colour mode must be one of" ) );
462 #endif
463 }
464 }
465 }
466