1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions
5 * are met:
6 * 1. Redistributions of source code must retain the above copyright
7 * notice, this list of conditions and the following disclaimer.
8 * 2. Redistributions in binary form must reproduce the above copyright
9 * notice, this list of conditions and the following disclaimer in the
10 * documentation and/or other materials provided with the distribution.
11 * 3. The name of the author may not be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25 #ifdef TINYTEST_LOCAL
26 #include "tinytest_local.h"
27 #endif
28
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <assert.h>
33
34 #ifndef NO_FORKING
35
36 #ifdef _WIN32
37 #include <windows.h>
38 #else
39 #include <sys/types.h>
40 #include <sys/wait.h>
41 #include <unistd.h>
42 #endif
43
44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47 /* Workaround for a stupid bug in OSX 10.6 */
48 #define FORK_BREAKS_GCOV
49 #include <vproc.h>
50 #endif
51 #endif
52
53 #endif /* !NO_FORKING */
54
55 #ifndef __GNUC__
56 #define __attribute__(x)
57 #endif
58
59 #include "tinytest.h"
60 #include "tinytest_macros.h"
61
62 #define LONGEST_TEST_NAME 16384
63
64 #ifndef _WIN32
65 #define DEFAULT_TESTCASE_TIMEOUT 30U
66 #else
67 #define DEFAULT_TESTCASE_TIMEOUT 0U
68 #endif
69
70 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
71 static int n_ok = 0; /**< Number of tests that have passed */
72 static int n_bad = 0; /**< Number of tests that have failed. */
73 static int n_skipped = 0; /**< Number of tests that have been skipped. */
74
75 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
76 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
77 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
78 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
79 const char *verbosity_flag = "";
80
81 const struct testlist_alias_t *cfg_aliases=NULL;
82
83 enum outcome { SKIP=2, OK=1, FAIL=0 };
84 static enum outcome cur_test_outcome = 0;
85 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
86 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
87 const char *cur_test_name = NULL;
88
89 #ifdef _WIN32
90 /* Copy of argv[0] for win32. */
91 static char commandname[MAX_PATH+1];
92 #endif
93
94 static void usage(struct testgroup_t *groups, int list_groups)
95 __attribute__((noreturn));
96 static int process_test_option(struct testgroup_t *groups, const char *test);
97
testcase_set_timeout_(void)98 static unsigned int testcase_set_timeout_(void)
99 {
100 if (!opt_timeout)
101 return 0;
102 #ifndef _WIN32
103 return alarm(opt_timeout);
104 #else
105 /** TODO: win32 support */
106 fprintf(stderr, "You cannot set alarm on windows\n");
107 exit(1);
108 #endif
109 }
testcase_reset_timeout_(void)110 static unsigned int testcase_reset_timeout_(void)
111 {
112 #ifndef _WIN32
113 return alarm(0);
114 #endif
115 }
116
117 static enum outcome
testcase_run_bare_(const struct testcase_t * testcase)118 testcase_run_bare_(const struct testcase_t *testcase)
119 {
120 void *env = NULL;
121 int outcome;
122 if (testcase->setup) {
123 env = testcase->setup->setup_fn(testcase);
124 if (!env)
125 return FAIL;
126 else if (env == (void*)TT_SKIP)
127 return SKIP;
128 }
129
130 cur_test_outcome = OK;
131 {
132 testcase_set_timeout_();
133 testcase->fn(env);
134 testcase_reset_timeout_();
135 }
136 outcome = cur_test_outcome;
137
138 if (testcase->setup) {
139 if (testcase->setup->cleanup_fn(testcase, env) == 0)
140 outcome = FAIL;
141 }
142
143 return outcome;
144 }
145
146 #define MAGIC_EXITCODE 42
147
148 #ifndef NO_FORKING
149
150 static enum outcome
testcase_run_forked_(const struct testgroup_t * group,const struct testcase_t * testcase)151 testcase_run_forked_(const struct testgroup_t *group,
152 const struct testcase_t *testcase)
153 {
154 #ifdef _WIN32
155 /* Fork? On Win32? How primitive! We'll do what the smart kids do:
156 we'll invoke our own exe (whose name we recall from the command
157 line) with a command line that tells it to run just the test we
158 want, and this time without forking.
159
160 (No, threads aren't an option. The whole point of forking is to
161 share no state between tests.)
162 */
163 int ok;
164 char buffer[LONGEST_TEST_NAME+256];
165 STARTUPINFOA si;
166 PROCESS_INFORMATION info;
167 DWORD exitcode;
168
169 if (!in_tinytest_main) {
170 printf("\nERROR. On Windows, testcase_run_forked_ must be"
171 " called from within tinytest_main.\n");
172 abort();
173 }
174 if (opt_verbosity>0)
175 printf("[forking] ");
176
177 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
178 commandname, verbosity_flag, group->prefix, testcase->name);
179
180 memset(&si, 0, sizeof(si));
181 memset(&info, 0, sizeof(info));
182 si.cb = sizeof(si);
183
184 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
185 0, NULL, NULL, &si, &info);
186 if (!ok) {
187 printf("CreateProcess failed!\n");
188 return 0;
189 }
190 WaitForSingleObject(info.hProcess, INFINITE);
191 GetExitCodeProcess(info.hProcess, &exitcode);
192 CloseHandle(info.hProcess);
193 CloseHandle(info.hThread);
194 if (exitcode == 0)
195 return OK;
196 else if (exitcode == MAGIC_EXITCODE)
197 return SKIP;
198 else
199 return FAIL;
200 #else
201 int outcome_pipe[2];
202 pid_t pid;
203 (void)group;
204
205 if (pipe(outcome_pipe))
206 perror("opening pipe");
207
208 if (opt_verbosity>0)
209 printf("[forking] ");
210 pid = fork();
211 #ifdef FORK_BREAKS_GCOV
212 vproc_transaction_begin(0);
213 #endif
214 if (!pid) {
215 /* child. */
216 int test_r, write_r;
217 char b[1];
218 close(outcome_pipe[0]);
219 test_r = testcase_run_bare_(testcase);
220 assert(0<=(int)test_r && (int)test_r<=2);
221 b[0] = "NYS"[test_r];
222 write_r = (int)write(outcome_pipe[1], b, 1);
223 if (write_r != 1) {
224 perror("write outcome to pipe");
225 exit(1);
226 }
227 exit(0);
228 return FAIL; /* unreachable */
229 } else {
230 /* parent */
231 int status, r;
232 char b[1];
233 /* Close this now, so that if the other side closes it,
234 * our read fails. */
235 close(outcome_pipe[1]);
236 r = (int)read(outcome_pipe[0], b, 1);
237 if (r == 0) {
238 printf("[Lost connection!] ");
239 return 0;
240 } else if (r != 1) {
241 perror("read outcome from pipe");
242 }
243 waitpid(pid, &status, 0);
244 close(outcome_pipe[0]);
245 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
246 }
247 #endif
248 }
249
250 #endif /* !NO_FORKING */
251
252 int
testcase_run_one(const struct testgroup_t * group,const struct testcase_t * testcase)253 testcase_run_one(const struct testgroup_t *group,
254 const struct testcase_t *testcase)
255 {
256 enum outcome outcome;
257
258 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
259 if (opt_verbosity>0)
260 printf("%s%s: %s\n",
261 group->prefix, testcase->name,
262 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
263 ++n_skipped;
264 return SKIP;
265 }
266
267 if (opt_verbosity>0 && !opt_forked) {
268 printf("%s%s: ", group->prefix, testcase->name);
269 } else {
270 if (opt_verbosity==0) printf(".");
271 cur_test_prefix = group->prefix;
272 cur_test_name = testcase->name;
273 }
274
275 #ifndef NO_FORKING
276 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
277 outcome = testcase_run_forked_(group, testcase);
278 } else {
279 #else
280 {
281 #endif
282 outcome = testcase_run_bare_(testcase);
283 }
284
285 if (outcome == OK) {
286 if (opt_verbosity>0 && !opt_forked)
287 puts(opt_verbosity==1?"OK":"");
288 } else if (outcome == SKIP) {
289 if (opt_verbosity>0 && !opt_forked)
290 puts("SKIPPED");
291 } else {
292 if (!opt_forked)
293 printf("\n [%s FAILED]\n", testcase->name);
294 }
295
296 if (opt_forked) {
297 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
298 return 1; /* unreachable */
299 } else {
300 return (int)outcome;
301 }
302 }
303
304 int
305 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
306 {
307 int i, j;
308 size_t length = LONGEST_TEST_NAME;
309 char fullname[LONGEST_TEST_NAME];
310 int found=0;
311 if (strstr(arg, ".."))
312 length = strstr(arg,"..")-arg;
313 for (i=0; groups[i].prefix; ++i) {
314 for (j=0; groups[i].cases[j].name; ++j) {
315 struct testcase_t *testcase = &groups[i].cases[j];
316 snprintf(fullname, sizeof(fullname), "%s%s",
317 groups[i].prefix, testcase->name);
318 if (!flag) { /* Hack! */
319 printf(" %s", fullname);
320 if (testcase->flags & TT_OFF_BY_DEFAULT)
321 puts(" (Off by default)");
322 else if (testcase->flags & TT_SKIP)
323 puts(" (DISABLED)");
324 else
325 puts("");
326 }
327 if (!strncmp(fullname, arg, length)) {
328 if (set)
329 testcase->flags |= flag;
330 else
331 testcase->flags &= ~flag;
332 ++found;
333 }
334 }
335 }
336 return found;
337 }
338
339 static void
340 usage(struct testgroup_t *groups, int list_groups)
341 {
342 puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
343 puts(" Specify tests by name, or using a prefix ending with '..'");
344 puts(" To skip a test, prefix its name with a colon.");
345 puts(" To enable a disabled test, prefix its name with a plus.");
346 puts(" Use --list-tests for a list of tests.");
347 if (list_groups) {
348 puts("Known tests are:");
349 tinytest_set_flag_(groups, "..", 1, 0);
350 }
351 exit(0);
352 }
353
354 static int
355 process_test_alias(struct testgroup_t *groups, const char *test)
356 {
357 int i, j, n, r;
358 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
359 if (!strcmp(cfg_aliases[i].name, test)) {
360 n = 0;
361 for (j = 0; cfg_aliases[i].tests[j]; ++j) {
362 r = process_test_option(groups, cfg_aliases[i].tests[j]);
363 if (r<0)
364 return -1;
365 n += r;
366 }
367 return n;
368 }
369 }
370 printf("No such test alias as @%s!",test);
371 return -1;
372 }
373
374 static int
375 process_test_option(struct testgroup_t *groups, const char *test)
376 {
377 int flag = TT_ENABLED_;
378 int n = 0;
379 if (test[0] == '@') {
380 return process_test_alias(groups, test + 1);
381 } else if (test[0] == ':') {
382 ++test;
383 flag = TT_SKIP;
384 } else if (test[0] == '+') {
385 ++test;
386 ++n;
387 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
388 printf("No such test as %s!\n", test);
389 return -1;
390 }
391 } else {
392 ++n;
393 }
394 if (!tinytest_set_flag_(groups, test, 1, flag)) {
395 printf("No such test as %s!\n", test);
396 return -1;
397 }
398 return n;
399 }
400
401 void
402 tinytest_set_aliases(const struct testlist_alias_t *aliases)
403 {
404 cfg_aliases = aliases;
405 }
406
407 int
408 tinytest_main(int c, const char **v, struct testgroup_t *groups)
409 {
410 int i, j, n=0;
411
412 #ifdef _WIN32
413 const char *sp = strrchr(v[0], '.');
414 const char *extension = "";
415 if (!sp || stricmp(sp, ".exe"))
416 extension = ".exe"; /* Add an exe so CreateProcess will work */
417 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
418 commandname[MAX_PATH]='\0';
419 #endif
420 for (i=1; i<c; ++i) {
421 if (v[i][0] == '-') {
422 if (!strcmp(v[i], "--RUNNING-FORKED")) {
423 opt_forked = 1;
424 } else if (!strcmp(v[i], "--no-fork")) {
425 opt_nofork = 1;
426 } else if (!strcmp(v[i], "--quiet")) {
427 opt_verbosity = -1;
428 verbosity_flag = "--quiet";
429 } else if (!strcmp(v[i], "--verbose")) {
430 opt_verbosity = 2;
431 verbosity_flag = "--verbose";
432 } else if (!strcmp(v[i], "--terse")) {
433 opt_verbosity = 0;
434 verbosity_flag = "--terse";
435 } else if (!strcmp(v[i], "--help")) {
436 usage(groups, 0);
437 } else if (!strcmp(v[i], "--list-tests")) {
438 usage(groups, 1);
439 } else if (!strcmp(v[i], "--timeout")) {
440 ++i;
441 if (i >= c) {
442 fprintf(stderr, "--timeout requires argument\n");
443 return -1;
444 }
445 opt_timeout = (unsigned)atoi(v[i]);
446 } else {
447 fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
448 return -1;
449 }
450 } else {
451 int r = process_test_option(groups, v[i]);
452 if (r<0)
453 return -1;
454 n += r;
455 }
456 }
457 if (!n)
458 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
459
460 #ifdef _IONBF
461 setvbuf(stdout, NULL, _IONBF, 0);
462 #endif
463
464 ++in_tinytest_main;
465 for (i = 0; groups[i].prefix; ++i) {
466 struct testgroup_t *group = &groups[i];
467 for (j = 0; group->cases[j].name; ++j) {
468 struct testcase_t *testcase = &group->cases[j];
469 int test_attempts = 3;
470 int test_ret_err;
471
472 if (!(testcase->flags & TT_ENABLED_))
473 continue;
474
475 for (;;) {
476 test_ret_err = testcase_run_one(group, testcase);
477
478 if (test_ret_err == OK)
479 break;
480 if (!(testcase->flags & TT_RETRIABLE))
481 break;
482 printf("\n [RETRYING %s (%i)]\n", testcase->name, test_attempts);
483 if (!test_attempts--)
484 break;
485 }
486
487 switch (test_ret_err) {
488 case OK: ++n_ok; break;
489 case SKIP: ++n_skipped; break;
490 default: ++n_bad; break;
491 }
492 }
493 }
494
495 --in_tinytest_main;
496
497 if (opt_verbosity==0)
498 puts("");
499
500 if (n_bad)
501 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
502 n_bad+n_ok,n_skipped);
503 else if (opt_verbosity >= 1)
504 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
505
506 return (n_bad == 0) ? 0 : 1;
507 }
508
509 int
510 tinytest_get_verbosity_(void)
511 {
512 return opt_verbosity;
513 }
514
515 void
516 tinytest_set_test_failed_(void)
517 {
518 if (opt_verbosity <= 0 && cur_test_name) {
519 if (opt_verbosity==0) puts("");
520 printf("%s%s: ", cur_test_prefix, cur_test_name);
521 cur_test_name = NULL;
522 }
523 cur_test_outcome = 0;
524 }
525
526 void
527 tinytest_set_test_skipped_(void)
528 {
529 if (cur_test_outcome==OK)
530 cur_test_outcome = SKIP;
531 }
532
533 char *
534 tinytest_format_hex_(const void *val_, unsigned long len)
535 {
536 const unsigned char *val = val_;
537 char *result, *cp;
538 size_t i;
539
540 if (!val)
541 return strdup("null");
542 if (!(result = malloc(len*2+1)))
543 return strdup("<allocation failure>");
544 cp = result;
545 for (i=0;i<len;++i) {
546 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
547 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
548 }
549 *cp = 0;
550 return result;
551 }
552