1 /*
2 * Created by Phil on 13/5/2013.
3 * Copyright 2014 Two Blue Cubes Ltd. All rights reserved.
4 *
5 * Distributed under the Boost Software License, Version 1.0. (See accompanying
6 * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 */
8
9 #include "catch.hpp"
10 #include "internal/catch_test_spec_parser.h"
11 #include "internal/catch_config.hpp"
12 #include "internal/catch_commandline.h"
13
14 #ifdef __clang__
15 # pragma clang diagnostic ignored "-Wc++98-compat"
16 #endif
17
fakeTestCase(const char * name,const char * desc="")18 inline Catch::TestCase fakeTestCase(const char* name, const char* desc = "") { return Catch::makeTestCase(nullptr, "", { name, desc }, CATCH_INTERNAL_LINEINFO); }
19
20 TEST_CASE( "Parse test names and tags", "[command-line][test-spec]" ) {
21
22 using Catch::parseTestSpec;
23 using Catch::TestSpec;
24
25 Catch::TestCase tcA = fakeTestCase( "a" );
26 Catch::TestCase tcB = fakeTestCase( "b", "[one][x]" );
27 Catch::TestCase tcC = fakeTestCase( "longer name with spaces", "[two][three][.][x]" );
28 Catch::TestCase tcD = fakeTestCase( "zlonger name with spacesz" );
29
30 SECTION( "Empty test spec should have no filters" ) {
31 TestSpec spec;
32 CHECK( spec.hasFilters() == false );
33 CHECK( spec.matches( tcA ) == false );
34 CHECK( spec.matches( tcB ) == false );
35 }
36
37 SECTION( "Test spec from empty string should have no filters" ) {
38 TestSpec spec = parseTestSpec( "" );
39 CHECK( spec.hasFilters() == false );
40 CHECK( spec.matches(tcA ) == false );
41 CHECK( spec.matches( tcB ) == false );
42 }
43
44 SECTION( "Test spec from just a comma should have no filters" ) {
45 TestSpec spec = parseTestSpec( "," );
46 CHECK( spec.hasFilters() == false );
47 CHECK( spec.matches( tcA ) == false );
48 CHECK( spec.matches( tcB ) == false );
49 }
50
51 SECTION( "Test spec from name should have one filter" ) {
52 TestSpec spec = parseTestSpec( "b" );
53 CHECK( spec.hasFilters() == true );
54 CHECK( spec.matches( tcA ) == false );
55 CHECK( spec.matches( tcB ) == true );
56 }
57
58 SECTION( "Test spec from quoted name should have one filter" ) {
59 TestSpec spec = parseTestSpec( "\"b\"" );
60 CHECK( spec.hasFilters() == true );
61 CHECK( spec.matches( tcA ) == false );
62 CHECK( spec.matches( tcB ) == true );
63 }
64
65 SECTION( "Test spec from name should have one filter" ) {
66 TestSpec spec = parseTestSpec( "b" );
67 CHECK( spec.hasFilters() == true );
68 CHECK( spec.matches( tcA ) == false );
69 CHECK( spec.matches( tcB ) == true );
70 CHECK( spec.matches( tcC ) == false );
71 }
72
73 SECTION( "Wildcard at the start" ) {
74 TestSpec spec = parseTestSpec( "*spaces" );
75 CHECK( spec.hasFilters() == true );
76 CHECK( spec.matches( tcA ) == false );
77 CHECK( spec.matches( tcB ) == false );
78 CHECK( spec.matches( tcC ) == true );
79 CHECK( spec.matches( tcD ) == false );
80 CHECK( parseTestSpec( "*a" ).matches( tcA ) == true );
81 }
82 SECTION( "Wildcard at the end" ) {
83 TestSpec spec = parseTestSpec( "long*" );
84 CHECK( spec.hasFilters() == true );
85 CHECK( spec.matches( tcA ) == false );
86 CHECK( spec.matches( tcB ) == false );
87 CHECK( spec.matches( tcC ) == true );
88 CHECK( spec.matches( tcD ) == false );
89 CHECK( parseTestSpec( "a*" ).matches( tcA ) == true );
90 }
91 SECTION( "Wildcard at both ends" ) {
92 TestSpec spec = parseTestSpec( "*name*" );
93 CHECK( spec.hasFilters() == true );
94 CHECK( spec.matches( tcA ) == false );
95 CHECK( spec.matches( tcB ) == false );
96 CHECK( spec.matches( tcC ) == true );
97 CHECK( spec.matches( tcD ) == true );
98 CHECK( parseTestSpec( "*a*" ).matches( tcA ) == true );
99 }
100 SECTION( "Redundant wildcard at the start" ) {
101 TestSpec spec = parseTestSpec( "*a" );
102 CHECK( spec.hasFilters() == true );
103 CHECK( spec.matches( tcA ) == true );
104 CHECK( spec.matches( tcB ) == false );
105 }
106 SECTION( "Redundant wildcard at the end" ) {
107 TestSpec spec = parseTestSpec( "a*" );
108 CHECK( spec.hasFilters() == true );
109 CHECK( spec.matches( tcA ) == true );
110 CHECK( spec.matches( tcB ) == false );
111 }
112 SECTION( "Redundant wildcard at both ends" ) {
113 TestSpec spec = parseTestSpec( "*a*" );
114 CHECK( spec.hasFilters() == true );
115 CHECK( spec.matches( tcA ) == true );
116 CHECK( spec.matches( tcB ) == false );
117 }
118 SECTION( "Wildcard at both ends, redundant at start" ) {
119 TestSpec spec = parseTestSpec( "*longer*" );
120 CHECK( spec.hasFilters() == true );
121 CHECK( spec.matches( tcA ) == false );
122 CHECK( spec.matches( tcB ) == false );
123 CHECK( spec.matches( tcC ) == true );
124 CHECK( spec.matches( tcD ) == true );
125 }
126 SECTION( "Just wildcard" ) {
127 TestSpec spec = parseTestSpec( "*" );
128 CHECK( spec.hasFilters() == true );
129 CHECK( spec.matches( tcA ) == true );
130 CHECK( spec.matches( tcB ) == true );
131 CHECK( spec.matches( tcC ) == true );
132 CHECK( spec.matches( tcD ) == true );
133 }
134
135 SECTION( "Single tag" ) {
136 TestSpec spec = parseTestSpec( "[one]" );
137 CHECK( spec.hasFilters() == true );
138 CHECK( spec.matches( tcA ) == false );
139 CHECK( spec.matches( tcB ) == true );
140 CHECK( spec.matches( tcC ) == false );
141 }
142 SECTION( "Single tag, two matches" ) {
143 TestSpec spec = parseTestSpec( "[x]" );
144 CHECK( spec.hasFilters() == true );
145 CHECK( spec.matches( tcA ) == false );
146 CHECK( spec.matches( tcB ) == true );
147 CHECK( spec.matches( tcC ) == true );
148 }
149 SECTION( "Two tags" ) {
150 TestSpec spec = parseTestSpec( "[two][x]" );
151 CHECK( spec.hasFilters() == true );
152 CHECK( spec.matches( tcA ) == false );
153 CHECK( spec.matches( tcB ) == false );
154 CHECK( spec.matches( tcC ) == true );
155 }
156 SECTION( "Two tags, spare separated" ) {
157 TestSpec spec = parseTestSpec( "[two] [x]" );
158 CHECK( spec.hasFilters() == true );
159 CHECK( spec.matches( tcA ) == false );
160 CHECK( spec.matches( tcB ) == false );
161 CHECK( spec.matches( tcC ) == true );
162 }
163 SECTION( "Wildcarded name and tag" ) {
164 TestSpec spec = parseTestSpec( "*name*[x]" );
165 CHECK( spec.hasFilters() == true );
166 CHECK( spec.matches( tcA ) == false );
167 CHECK( spec.matches( tcB ) == false );
168 CHECK( spec.matches( tcC ) == true );
169 CHECK( spec.matches( tcD ) == false );
170 }
171 SECTION( "Single tag exclusion" ) {
172 TestSpec spec = parseTestSpec( "~[one]" );
173 CHECK( spec.hasFilters() == true );
174 CHECK( spec.matches( tcA ) == true );
175 CHECK( spec.matches( tcB ) == false );
176 CHECK( spec.matches( tcC ) == true );
177 }
178 SECTION( "One tag exclusion and one tag inclusion" ) {
179 TestSpec spec = parseTestSpec( "~[two][x]" );
180 CHECK( spec.hasFilters() == true );
181 CHECK( spec.matches( tcA ) == false );
182 CHECK( spec.matches( tcB ) == true );
183 CHECK( spec.matches( tcC ) == false );
184 }
185 SECTION( "One tag exclusion and one wldcarded name inclusion" ) {
186 TestSpec spec = parseTestSpec( "~[two]*name*" );
187 CHECK( spec.hasFilters() == true );
188 CHECK( spec.matches( tcA ) == false );
189 CHECK( spec.matches( tcB ) == false );
190 CHECK( spec.matches( tcC ) == false );
191 CHECK( spec.matches( tcD ) == true );
192 }
193 SECTION( "One tag exclusion, using exclude:, and one wldcarded name inclusion" ) {
194 TestSpec spec = parseTestSpec( "exclude:[two]*name*" );
195 CHECK( spec.hasFilters() == true );
196 CHECK( spec.matches( tcA ) == false );
197 CHECK( spec.matches( tcB ) == false );
198 CHECK( spec.matches( tcC ) == false );
199 CHECK( spec.matches( tcD ) == true );
200 }
201 SECTION( "name exclusion" ) {
202 TestSpec spec = parseTestSpec( "~b" );
203 CHECK( spec.hasFilters() == true );
204 CHECK( spec.matches( tcA ) == true );
205 CHECK( spec.matches( tcB ) == false );
206 CHECK( spec.matches( tcC ) == true );
207 CHECK( spec.matches( tcD ) == true );
208 }
209 SECTION( "wildcarded name exclusion" ) {
210 TestSpec spec = parseTestSpec( "~*name*" );
211 CHECK( spec.hasFilters() == true );
212 CHECK( spec.matches( tcA ) == true );
213 CHECK( spec.matches( tcB ) == true );
214 CHECK( spec.matches( tcC ) == false );
215 CHECK( spec.matches( tcD ) == false );
216 }
217 SECTION( "wildcarded name exclusion with tag inclusion" ) {
218 TestSpec spec = parseTestSpec( "~*name*,[three]" );
219 CHECK( spec.hasFilters() == true );
220 CHECK( spec.matches( tcA ) == true );
221 CHECK( spec.matches( tcB ) == true );
222 CHECK( spec.matches( tcC ) == true );
223 CHECK( spec.matches( tcD ) == false );
224 }
225 SECTION( "wildcarded name exclusion, using exclude:, with tag inclusion" ) {
226 TestSpec spec = parseTestSpec( "exclude:*name*,[three]" );
227 CHECK( spec.hasFilters() == true );
228 CHECK( spec.matches( tcA ) == true );
229 CHECK( spec.matches( tcB ) == true );
230 CHECK( spec.matches( tcC ) == true );
231 CHECK( spec.matches( tcD ) == false );
232 }
233 SECTION( "two wildcarded names" ) {
234 TestSpec spec = parseTestSpec( "\"longer*\"\"*spaces\"" );
235 CHECK( spec.hasFilters() == true );
236 CHECK( spec.matches( tcA ) == false );
237 CHECK( spec.matches( tcB ) == false );
238 CHECK( spec.matches( tcC ) == true );
239 CHECK( spec.matches( tcD ) == false );
240 }
241 SECTION( "empty tag" ) {
242 TestSpec spec = parseTestSpec( "[]" );
243 CHECK( spec.hasFilters() == false );
244 CHECK( spec.matches( tcA ) == false );
245 CHECK( spec.matches( tcB ) == false );
246 CHECK( spec.matches( tcC ) == false );
247 CHECK( spec.matches( tcD ) == false );
248 }
249 SECTION( "empty quoted name" ) {
250 TestSpec spec = parseTestSpec( "\"\"" );
251 CHECK( spec.hasFilters() == false );
252 CHECK( spec.matches( tcA ) == false );
253 CHECK( spec.matches( tcB ) == false );
254 CHECK( spec.matches( tcC ) == false );
255 CHECK( spec.matches( tcD ) == false );
256 }
257 SECTION( "quoted string followed by tag exclusion" ) {
258 TestSpec spec = parseTestSpec( "\"*name*\"~[.]" );
259 CHECK( spec.hasFilters() == true );
260 CHECK( spec.matches( tcA ) == false );
261 CHECK( spec.matches( tcB ) == false );
262 CHECK( spec.matches( tcC ) == false );
263 CHECK( spec.matches( tcD ) == true );
264 }
265 SECTION( "Leading and trailing spaces in test spec" ) {
266 TestSpec spec = parseTestSpec( "\" aardvark \"" );
267 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
268 CHECK( spec.matches( fakeTestCase( " aardvark" ) ) );
269 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
270 CHECK( spec.matches( fakeTestCase( "aardvark " ) ) );
271 CHECK( spec.matches( fakeTestCase( "aardvark" ) ) );
272 }
273 SECTION( "Leading and trailing spaces in test name" ) {
274 TestSpec spec = parseTestSpec( "aardvark" );
275 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
276 CHECK( spec.matches( fakeTestCase( " aardvark" ) ) );
277 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
278 CHECK( spec.matches( fakeTestCase( "aardvark " ) ) );
279 CHECK( spec.matches( fakeTestCase( "aardvark" ) ) );
280 }
281 SECTION("Shortened hide tags are split apart when parsing") {
282 TestSpec spec = parseTestSpec("[.foo]");
283 CHECK(spec.matches(fakeTestCase("hidden and foo", "[.][foo]")));
284 CHECK_FALSE(spec.matches(fakeTestCase("only foo", "[foo]")));
285 }
286 SECTION("Shortened hide tags also properly handle exclusion") {
287 TestSpec spec = parseTestSpec("~[.foo]");
288 CHECK_FALSE(spec.matches(fakeTestCase("hidden and foo", "[.][foo]")));
289 CHECK_FALSE(spec.matches(fakeTestCase("only foo", "[foo]")));
290 CHECK_FALSE(spec.matches(fakeTestCase("only hidden", "[.]")));
291 CHECK(spec.matches(fakeTestCase("neither foo nor hidden", "[bar]")));
292 }
293 }
294
295 TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {
296
297 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
298 using namespace Catch::Matchers;
299 #endif
300
301 Catch::ConfigData config;
302 auto cli = Catch::makeCommandLineParser(config);
303
304 SECTION("empty args don't cause a crash") {
305 auto result = cli.parse({""});
306 CHECK(result);
307 CHECK(config.processName == "");
308 }
309
310 SECTION("default - no arguments") {
311 auto result = cli.parse({"test"});
312 CHECK(result);
313 CHECK(config.processName == "test");
314 CHECK(config.shouldDebugBreak == false);
315 CHECK(config.abortAfter == -1);
316 CHECK(config.noThrow == false);
317 CHECK(config.reporterName == "console");
318
319 Catch::Config cfg(config);
320 CHECK_FALSE(cfg.hasTestFilters());
321 }
322
323 SECTION("test lists") {
324 SECTION("Specify one test case using") {
325 auto result = cli.parse({"test", "test1"});
326 CHECK(result);
327
328 Catch::Config cfg(config);
329 REQUIRE(cfg.hasTestFilters());
330 REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
331 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
332 }
333 SECTION("Specify one test case exclusion using exclude:") {
334 auto result = cli.parse({"test", "exclude:test1"});
335 CHECK(result);
336
337 Catch::Config cfg(config);
338 REQUIRE(cfg.hasTestFilters());
339 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
340 REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
341 }
342
343 SECTION("Specify one test case exclusion using ~") {
344 auto result = cli.parse({"test", "~test1"});
345 CHECK(result);
346
347 Catch::Config cfg(config);
348 REQUIRE(cfg.hasTestFilters());
349 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
350 REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
351 }
352
353 }
354
355 SECTION("reporter") {
356 SECTION("-r/console") {
357 CHECK(cli.parse({"test", "-r", "console"}));
358
359 REQUIRE(config.reporterName == "console");
360 }
361 SECTION("-r/xml") {
362 CHECK(cli.parse({"test", "-r", "xml"}));
363
364 REQUIRE(config.reporterName == "xml");
365 }
366 SECTION("--reporter/junit") {
367 CHECK(cli.parse({"test", "--reporter", "junit"}));
368
369 REQUIRE(config.reporterName == "junit");
370 }
371 SECTION("Only one reporter is accepted") {
372 REQUIRE_FALSE(cli.parse({ "test", "-r", "xml", "-r", "junit" }));
373 }
374 SECTION("must match one of the available ones") {
375 auto result = cli.parse({"test", "--reporter", "unsupported"});
376 CHECK(!result);
377
378 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
379 REQUIRE_THAT(result.errorMessage(), Contains("Unrecognized reporter"));
380 #endif
381 }
382 }
383
384 SECTION("debugger") {
385 SECTION("-b") {
386 CHECK(cli.parse({"test", "-b"}));
387
388 REQUIRE(config.shouldDebugBreak == true);
389 }
390 SECTION("--break") {
391 CHECK(cli.parse({"test", "--break"}));
392
393 REQUIRE(config.shouldDebugBreak);
394 }
395 }
396
397
398 SECTION("abort") {
399 SECTION("-a aborts after first failure") {
400 CHECK(cli.parse({"test", "-a"}));
401
402 REQUIRE(config.abortAfter == 1);
403 }
404 SECTION("-x 2 aborts after two failures") {
405 CHECK(cli.parse({"test", "-x", "2"}));
406
407 REQUIRE(config.abortAfter == 2);
408 }
409 SECTION("-x must be numeric") {
410 auto result = cli.parse({"test", "-x", "oops"});
411 CHECK(!result);
412
413 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
414 REQUIRE_THAT(result.errorMessage(), Contains("convert") && Contains("oops"));
415 #endif
416 }
417
418 SECTION("wait-for-keypress") {
419 SECTION("Accepted options") {
420 using tuple_type = std::tuple<char const*, Catch::WaitForKeypress::When>;
421 auto input = GENERATE(table<char const*, Catch::WaitForKeypress::When>({
422 tuple_type{"never", Catch::WaitForKeypress::Never},
423 tuple_type{"start", Catch::WaitForKeypress::BeforeStart},
424 tuple_type{"exit", Catch::WaitForKeypress::BeforeExit},
425 tuple_type{"both", Catch::WaitForKeypress::BeforeStartAndExit},
426 }));
427 CHECK(cli.parse({"test", "--wait-for-keypress", std::get<0>(input)}));
428
429 REQUIRE(config.waitForKeypress == std::get<1>(input));
430 }
431
432 SECTION("invalid options are reported") {
433 auto result = cli.parse({"test", "--wait-for-keypress", "sometimes"});
434 CHECK(!result);
435
436 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
437 REQUIRE_THAT(result.errorMessage(), Contains("never") && Contains("both"));
438 #endif
439 }
440 }
441 }
442
443 SECTION("nothrow") {
444 SECTION("-e") {
445 CHECK(cli.parse({"test", "-e"}));
446
447 REQUIRE(config.noThrow);
448 }
449 SECTION("--nothrow") {
450 CHECK(cli.parse({"test", "--nothrow"}));
451
452 REQUIRE(config.noThrow);
453 }
454 }
455
456 SECTION("output filename") {
457 SECTION("-o filename") {
458 CHECK(cli.parse({"test", "-o", "filename.ext"}));
459
460 REQUIRE(config.outputFilename == "filename.ext");
461 }
462 SECTION("--out") {
463 CHECK(cli.parse({"test", "--out", "filename.ext"}));
464
465 REQUIRE(config.outputFilename == "filename.ext");
466 }
467 }
468
469 SECTION("combinations") {
470 SECTION("Single character flags can be combined") {
471 CHECK(cli.parse({"test", "-abe"}));
472
473 CHECK(config.abortAfter == 1);
474 CHECK(config.shouldDebugBreak);
475 CHECK(config.noThrow == true);
476 }
477 }
478
479
480 SECTION( "use-colour") {
481
482 using Catch::UseColour;
483
484 SECTION( "without option" ) {
485 CHECK(cli.parse({"test"}));
486
487 REQUIRE( config.useColour == UseColour::Auto );
488 }
489
490 SECTION( "auto" ) {
491 CHECK(cli.parse({"test", "--use-colour", "auto"}));
492
493 REQUIRE( config.useColour == UseColour::Auto );
494 }
495
496 SECTION( "yes" ) {
497 CHECK(cli.parse({"test", "--use-colour", "yes"}));
498
499 REQUIRE( config.useColour == UseColour::Yes );
500 }
501
502 SECTION( "no" ) {
503 CHECK(cli.parse({"test", "--use-colour", "no"}));
504
505 REQUIRE( config.useColour == UseColour::No );
506 }
507
508 SECTION( "error" ) {
509 auto result = cli.parse({"test", "--use-colour", "wrong"});
510 CHECK( !result );
511 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
512 CHECK_THAT( result.errorMessage(), Contains( "colour mode must be one of" ) );
513 #endif
514 }
515 }
516
517 SECTION("Benchmark options") {
518 SECTION("samples") {
519 CHECK(cli.parse({ "test", "--benchmark-samples=200" }));
520
521 REQUIRE(config.benchmarkSamples == 200);
522 }
523
524 SECTION("resamples") {
525 CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));
526
527 REQUIRE(config.benchmarkResamples == 20000);
528 }
529
530 SECTION("confidence-interval") {
531 CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
532
533 REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99));
534 }
535
536 SECTION("no-analysis") {
537 CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
538
539 REQUIRE(config.benchmarkNoAnalysis);
540 }
541
542 SECTION("warmup-time") {
543 CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" }));
544
545 REQUIRE(config.benchmarkWarmupTime == 10);
546 }
547 }
548 }
549
550 TEST_CASE("Test with special, characters \"in name", "[cli][regression]") {
551 // This test case succeeds if we can invoke it from the CLI
552 SUCCEED();
553 }
554