1 /*
2 * Copyright 2016-2022 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10 #include "../testutil.h"
11 #include "output.h"
12 #include "tu_local.h"
13
14 #include <string.h>
15 #include <assert.h>
16
17 #include "internal/nelem.h"
18 #include <openssl/bio.h>
19
20 #include "platform.h" /* From libapps */
21
22 #if defined(_WIN32) && !defined(__BORLANDC__)
23 # define strdup _strdup
24 #endif
25
26
27 /*
28 * Declares the structures needed to register each test case function.
29 */
30 typedef struct test_info {
31 const char *test_case_name;
32 int (*test_fn) (void);
33 int (*param_test_fn)(int idx);
34 int num;
35
36 /* flags */
37 int subtest:1;
38 } TEST_INFO;
39
40 static TEST_INFO all_tests[1024];
41 static int num_tests = 0;
42 static int show_list = 0;
43 static int single_test = -1;
44 static int single_iter = -1;
45 static int level = 0;
46 static int seed = 0;
47 static int rand_order = 0;
48
49 /*
50 * A parameterised test runs a loop of test cases.
51 * |num_test_cases| counts the total number of non-subtest test cases
52 * across all tests.
53 */
54 static int num_test_cases = 0;
55
56 static int process_shared_options(void);
57
58
add_test(const char * test_case_name,int (* test_fn)(void))59 void add_test(const char *test_case_name, int (*test_fn) (void))
60 {
61 assert(num_tests != OSSL_NELEM(all_tests));
62 all_tests[num_tests].test_case_name = test_case_name;
63 all_tests[num_tests].test_fn = test_fn;
64 all_tests[num_tests].num = -1;
65 ++num_tests;
66 ++num_test_cases;
67 }
68
add_all_tests(const char * test_case_name,int (* test_fn)(int idx),int num,int subtest)69 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
70 int num, int subtest)
71 {
72 assert(num_tests != OSSL_NELEM(all_tests));
73 all_tests[num_tests].test_case_name = test_case_name;
74 all_tests[num_tests].param_test_fn = test_fn;
75 all_tests[num_tests].num = num;
76 all_tests[num_tests].subtest = subtest;
77 ++num_tests;
78 if (subtest)
79 ++num_test_cases;
80 else
81 num_test_cases += num;
82 }
83
gcd(int a,int b)84 static int gcd(int a, int b)
85 {
86 while (b != 0) {
87 int t = b;
88 b = a % b;
89 a = t;
90 }
91 return a;
92 }
93
set_seed(int s)94 static void set_seed(int s)
95 {
96 seed = s;
97 if (seed <= 0)
98 seed = (int)time(NULL);
99 test_random_seed(seed);
100 }
101
102
setup_test_framework(int argc,char * argv[])103 int setup_test_framework(int argc, char *argv[])
104 {
105 char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
106 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
107
108 if (TAP_levels != NULL)
109 level = 4 * atoi(TAP_levels);
110 test_adjust_streams_tap_level(level);
111 if (test_seed != NULL) {
112 rand_order = 1;
113 set_seed(atoi(test_seed));
114 } else {
115 set_seed(0);
116 }
117
118 #if defined(OPENSSL_SYS_VMS) && defined(__DECC)
119 argv = copy_argv(&argc, argv);
120 #elif defined(_WIN32)
121 /*
122 * Replace argv[] with UTF-8 encoded strings.
123 */
124 win32_utf8argv(&argc, &argv);
125 #endif
126
127 if (!opt_init(argc, argv, test_get_options()))
128 return 0;
129 return 1;
130 }
131
132
133 /*
134 * This can only be called after setup() has run, since num_tests and
135 * all_tests[] are setup at this point
136 */
check_single_test_params(char * name,char * testname,char * itname)137 static int check_single_test_params(char *name, char *testname, char *itname)
138 {
139 if (name != NULL) {
140 int i;
141 for (i = 0; i < num_tests; ++i) {
142 if (strcmp(name, all_tests[i].test_case_name) == 0) {
143 single_test = 1 + i;
144 break;
145 }
146 }
147 if (i >= num_tests)
148 single_test = atoi(name);
149 }
150
151
152 /* if only iteration is specified, assume we want the first test */
153 if (single_test == -1 && single_iter != -1)
154 single_test = 1;
155
156 if (single_test != -1) {
157 if (single_test < 1 || single_test > num_tests) {
158 test_printf_stderr("Invalid -%s value "
159 "(Value must be a valid test name OR a value between %d..%d)\n",
160 testname, 1, num_tests);
161 return 0;
162 }
163 }
164 if (single_iter != -1) {
165 if (all_tests[single_test - 1].num == -1) {
166 test_printf_stderr("-%s option is not valid for test %d:%s\n",
167 itname,
168 single_test,
169 all_tests[single_test - 1].test_case_name);
170 return 0;
171 } else if (single_iter < 1
172 || single_iter > all_tests[single_test - 1].num) {
173 test_printf_stderr("Invalid -%s value for test %d:%s\t"
174 "(Value must be in the range %d..%d)\n",
175 itname, single_test,
176 all_tests[single_test - 1].test_case_name,
177 1, all_tests[single_test - 1].num);
178 return 0;
179 }
180 }
181 return 1;
182 }
183
process_shared_options(void)184 static int process_shared_options(void)
185 {
186 OPTION_CHOICE_DEFAULT o;
187 int value;
188 int ret = -1;
189 char *flag_test = "";
190 char *flag_iter = "";
191 char *testname = NULL;
192
193 opt_begin();
194 while ((o = opt_next()) != OPT_EOF) {
195 switch (o) {
196 /* Ignore any test options at this level */
197 default:
198 break;
199 case OPT_ERR:
200 return ret;
201 case OPT_TEST_HELP:
202 opt_help(test_get_options());
203 return 0;
204 case OPT_TEST_LIST:
205 show_list = 1;
206 break;
207 case OPT_TEST_SINGLE:
208 flag_test = opt_flag();
209 testname = opt_arg();
210 break;
211 case OPT_TEST_ITERATION:
212 flag_iter = opt_flag();
213 if (!opt_int(opt_arg(), &single_iter))
214 goto end;
215 break;
216 case OPT_TEST_INDENT:
217 if (!opt_int(opt_arg(), &value))
218 goto end;
219 level = 4 * value;
220 test_adjust_streams_tap_level(level);
221 break;
222 case OPT_TEST_SEED:
223 if (!opt_int(opt_arg(), &value))
224 goto end;
225 set_seed(value);
226 break;
227 }
228 }
229 if (!check_single_test_params(testname, flag_test, flag_iter))
230 goto end;
231 ret = 1;
232 end:
233 return ret;
234 }
235
236
pulldown_test_framework(int ret)237 int pulldown_test_framework(int ret)
238 {
239 set_test_title(NULL);
240 return ret;
241 }
242
finalize(int success)243 static void finalize(int success)
244 {
245 if (success)
246 ERR_clear_error();
247 else
248 ERR_print_errors_cb(openssl_error_cb, NULL);
249 }
250
251 static char *test_title = NULL;
252
set_test_title(const char * title)253 void set_test_title(const char *title)
254 {
255 free(test_title);
256 test_title = title == NULL ? NULL : strdup(title);
257 }
258
test_verdict(int verdict,const char * description,...)259 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
260 const char *description, ...)
261 {
262 va_list ap;
263
264 test_flush_stdout();
265 test_flush_stderr();
266
267 if (verdict == 0 && seed != 0)
268 test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed);
269 test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok");
270 va_start(ap, description);
271 test_vprintf_tapout(description, ap);
272 va_end(ap);
273 if (verdict == TEST_SKIP_CODE)
274 test_printf_tapout(" # skipped");
275 test_printf_tapout("\n");
276 test_flush_tapout();
277 }
278
run_tests(const char * test_prog_name)279 int run_tests(const char *test_prog_name)
280 {
281 int num_failed = 0;
282 int verdict = 1;
283 int ii, i, jj, j, jstep;
284 int test_case_count = 0;
285 int subtest_case_count = 0;
286 int permute[OSSL_NELEM(all_tests)];
287
288 i = process_shared_options();
289 if (i == 0)
290 return EXIT_SUCCESS;
291 if (i == -1)
292 return EXIT_FAILURE;
293
294 if (num_tests < 1) {
295 test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name);
296 } else if (show_list == 0 && single_test == -1) {
297 if (level > 0) {
298 test_printf_stdout("Subtest: %s\n", test_prog_name);
299 test_flush_stdout();
300 }
301 test_printf_tapout("1..%d\n", num_test_cases);
302 }
303
304 test_flush_tapout();
305
306 for (i = 0; i < num_tests; i++)
307 permute[i] = i;
308 if (rand_order != 0)
309 for (i = num_tests - 1; i >= 1; i--) {
310 j = test_random() % (1 + i);
311 ii = permute[j];
312 permute[j] = permute[i];
313 permute[i] = ii;
314 }
315
316 for (ii = 0; ii != num_tests; ++ii) {
317 i = permute[ii];
318
319 if (single_test != -1 && ((i+1) != single_test)) {
320 continue;
321 }
322 else if (show_list) {
323 if (all_tests[i].num != -1) {
324 test_printf_tapout("%d - %s (%d..%d)\n", ii + 1,
325 all_tests[i].test_case_name, 1,
326 all_tests[i].num);
327 } else {
328 test_printf_tapout("%d - %s\n", ii + 1,
329 all_tests[i].test_case_name);
330 }
331 test_flush_tapout();
332 } else if (all_tests[i].num == -1) {
333 set_test_title(all_tests[i].test_case_name);
334 ERR_clear_error();
335 verdict = all_tests[i].test_fn();
336 finalize(verdict != 0);
337 test_verdict(verdict, "%d - %s", test_case_count + 1, test_title);
338 if (verdict == 0)
339 num_failed++;
340 test_case_count++;
341 } else {
342 verdict = TEST_SKIP_CODE;
343 set_test_title(all_tests[i].test_case_name);
344 if (all_tests[i].subtest) {
345 level += 4;
346 test_adjust_streams_tap_level(level);
347 if (single_iter == -1) {
348 test_printf_stdout("Subtest: %s\n", test_title);
349 test_printf_tapout("%d..%d\n", 1, all_tests[i].num);
350 test_flush_stdout();
351 test_flush_tapout();
352 }
353 }
354
355 j = -1;
356 if (rand_order == 0 || all_tests[i].num < 3)
357 jstep = 1;
358 else
359 do
360 jstep = test_random() % all_tests[i].num;
361 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
362
363 for (jj = 0; jj < all_tests[i].num; jj++) {
364 int v;
365
366 j = (j + jstep) % all_tests[i].num;
367 if (single_iter != -1 && ((jj + 1) != single_iter))
368 continue;
369 ERR_clear_error();
370 v = all_tests[i].param_test_fn(j);
371
372 if (v == 0) {
373 verdict = 0;
374 } else if (v != TEST_SKIP_CODE && verdict != 0) {
375 verdict = 1;
376 }
377
378 finalize(v != 0);
379
380 if (all_tests[i].subtest)
381 test_verdict(v, "%d - iteration %d",
382 subtest_case_count + 1, j + 1);
383 else
384 test_verdict(v, "%d - %s - iteration %d",
385 test_case_count + subtest_case_count + 1,
386 test_title, j + 1);
387 subtest_case_count++;
388 }
389
390 if (all_tests[i].subtest) {
391 level -= 4;
392 test_adjust_streams_tap_level(level);
393 }
394 if (verdict == 0)
395 ++num_failed;
396 if (all_tests[i].num == -1 || all_tests[i].subtest)
397 test_verdict(verdict, "%d - %s", test_case_count + 1,
398 all_tests[i].test_case_name);
399 test_case_count++;
400 }
401 }
402 if (num_failed != 0)
403 return EXIT_FAILURE;
404 return EXIT_SUCCESS;
405 }
406
407 /*
408 * Glue an array of strings together and return it as an allocated string.
409 * Optionally return the whole length of this string in |out_len|
410 */
glue_strings(const char * list[],size_t * out_len)411 char *glue_strings(const char *list[], size_t *out_len)
412 {
413 size_t len = 0;
414 char *p, *ret;
415 int i;
416
417 for (i = 0; list[i] != NULL; i++)
418 len += strlen(list[i]);
419
420 if (out_len != NULL)
421 *out_len = len;
422
423 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
424 return NULL;
425
426 for (i = 0; list[i] != NULL; i++)
427 p += strlen(strcpy(p, list[i]));
428
429 return ret;
430 }
431
test_mk_file_path(const char * dir,const char * file)432 char *test_mk_file_path(const char *dir, const char *file)
433 {
434 # ifndef OPENSSL_SYS_VMS
435 const char *sep = "/";
436 # else
437 const char *sep = "";
438 char *dir_end;
439 char dir_end_sep;
440 # endif
441 size_t dirlen = dir != NULL ? strlen(dir) : 0;
442 size_t len = dirlen + strlen(sep) + strlen(file) + 1;
443 char *full_file = OPENSSL_zalloc(len);
444
445 if (full_file != NULL) {
446 if (dir != NULL && dirlen > 0) {
447 OPENSSL_strlcpy(full_file, dir, len);
448 # ifdef OPENSSL_SYS_VMS
449 /*
450 * If |file| contains a directory spec, we need to do some
451 * careful merging.
452 * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become
453 * "vol:[dir.dir.certs]sm2-root.crt"
454 */
455 dir_end = &full_file[strlen(full_file) - 1];
456 dir_end_sep = *dir_end;
457 if ((dir_end_sep == ']' || dir_end_sep == '>')
458 && (file[0] == '[' || file[0] == '<')) {
459 file++;
460 if (file[0] == '.')
461 *dir_end = '\0';
462 else
463 *dir_end = '.';
464 }
465 #else
466 OPENSSL_strlcat(full_file, sep, len);
467 #endif
468 }
469 OPENSSL_strlcat(full_file, file, len);
470 }
471
472 return full_file;
473 }
474