1#!/usr/bin/env python3 2 3# This script manages littlefs tests, which are configured with 4# .toml files stored in the tests directory. 5# 6 7import toml 8import glob 9import re 10import os 11import io 12import itertools as it 13import collections.abc as abc 14import subprocess as sp 15import base64 16import sys 17import copy 18import shlex 19import pty 20import errno 21import signal 22 23TEST_PATHS = 'tests' 24RULES = """ 25# add block devices to sources 26TESTSRC ?= $(SRC) $(wildcard bd/*.c) 27 28define FLATTEN 29%(path)s%%$(subst /,.,$(target)): $(target) 30 ./scripts/explode_asserts.py $$< -o $$@ 31endef 32$(foreach target,$(TESTSRC),$(eval $(FLATTEN))) 33 34-include %(path)s*.d 35.SECONDARY: 36 37%(path)s.test: %(path)s.test.o \\ 38 $(foreach t,$(subst /,.,$(TESTSRC:.c=.o)),%(path)s.$t) 39 $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ 40 41# needed in case builddir is different 42%(path)s%%.o: %(path)s%%.c 43 $(CC) -c -MMD $(CFLAGS) $< -o $@ 44""" 45COVERAGE_RULES = """ 46%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage 47 48# delete lingering coverage 49%(path)s.test: | %(path)s.info.clean 50.PHONY: %(path)s.info.clean 51%(path)s.info.clean: 52 rm -f %(path)s*.gcda 53 54# accumulate coverage info 55.PHONY: %(path)s.info 56%(path)s.info: 57 $(strip $(LCOV) -c \\ 58 $(addprefix -d ,$(wildcard %(path)s*.gcda)) \\ 59 --rc 'geninfo_adjust_src_path=$(shell pwd)' \\ 60 -o $@) 61 $(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@ 62ifdef COVERAGETARGET 63 $(strip $(LCOV) -a $@ \\ 64 $(addprefix -a ,$(wildcard $(COVERAGETARGET))) \\ 65 -o $(COVERAGETARGET)) 66endif 67""" 68GLOBALS = """ 69//////////////// AUTOGENERATED TEST //////////////// 70#include "lfs.h" 71#include "bd/lfs_testbd.h" 72#include <stdio.h> 73extern const char *lfs_testbd_path; 74extern uint32_t lfs_testbd_cycles; 75""" 76DEFINES = { 77 'LFS_READ_SIZE': 16, 78 'LFS_PROG_SIZE': 'LFS_READ_SIZE', 79 'LFS_BLOCK_SIZE': 512, 80 'LFS_BLOCK_COUNT': 1024, 81 'LFS_BLOCK_CYCLES': -1, 82 'LFS_CACHE_SIZE': '(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)', 83 'LFS_LOOKAHEAD_SIZE': 16, 84 'LFS_ERASE_VALUE': 0xff, 85 'LFS_ERASE_CYCLES': 0, 86 'LFS_BADBLOCK_BEHAVIOR': 'LFS_TESTBD_BADBLOCK_PROGERROR', 87} 88PROLOGUE = """ 89 // prologue 90 __attribute__((unused)) lfs_t lfs; 91 __attribute__((unused)) lfs_testbd_t bd; 92 __attribute__((unused)) lfs_file_t file; 93 __attribute__((unused)) lfs_dir_t dir; 94 __attribute__((unused)) struct lfs_info info; 95 __attribute__((unused)) char path[1024]; 96 __attribute__((unused)) uint8_t buffer[1024]; 97 __attribute__((unused)) lfs_size_t size; 98 __attribute__((unused)) int err; 99 100 __attribute__((unused)) const struct lfs_config cfg = { 101 .context = &bd, 102 .read = lfs_testbd_read, 103 .prog = lfs_testbd_prog, 104 .erase = lfs_testbd_erase, 105 .sync = lfs_testbd_sync, 106 .read_size = LFS_READ_SIZE, 107 .prog_size = LFS_PROG_SIZE, 108 .block_size = LFS_BLOCK_SIZE, 109 .block_count = LFS_BLOCK_COUNT, 110 .block_cycles = LFS_BLOCK_CYCLES, 111 .cache_size = LFS_CACHE_SIZE, 112 .lookahead_size = LFS_LOOKAHEAD_SIZE, 113 }; 114 115 __attribute__((unused)) const struct lfs_testbd_config bdcfg = { 116 .erase_value = LFS_ERASE_VALUE, 117 .erase_cycles = LFS_ERASE_CYCLES, 118 .badblock_behavior = LFS_BADBLOCK_BEHAVIOR, 119 .power_cycles = lfs_testbd_cycles, 120 }; 121 122 lfs_testbd_createcfg(&cfg, lfs_testbd_path, &bdcfg) => 0; 123""" 124EPILOGUE = """ 125 // epilogue 126 lfs_testbd_destroy(&cfg) => 0; 127""" 128PASS = '\033[32m✓\033[0m' 129FAIL = '\033[31m✗\033[0m' 130 131class TestFailure(Exception): 132 def __init__(self, case, returncode=None, stdout=None, assert_=None): 133 self.case = case 134 self.returncode = returncode 135 self.stdout = stdout 136 self.assert_ = assert_ 137 138class TestCase: 139 def __init__(self, config, filter=filter, 140 suite=None, caseno=None, lineno=None, **_): 141 self.config = config 142 self.filter = filter 143 self.suite = suite 144 self.caseno = caseno 145 self.lineno = lineno 146 147 self.code = config['code'] 148 self.code_lineno = config['code_lineno'] 149 self.defines = config.get('define', {}) 150 self.if_ = config.get('if', None) 151 self.in_ = config.get('in', None) 152 153 self.result = None 154 155 def __str__(self): 156 if hasattr(self, 'permno'): 157 if any(k not in self.case.defines for k in self.defines): 158 return '%s#%d#%d (%s)' % ( 159 self.suite.name, self.caseno, self.permno, ', '.join( 160 '%s=%s' % (k, v) for k, v in self.defines.items() 161 if k not in self.case.defines)) 162 else: 163 return '%s#%d#%d' % ( 164 self.suite.name, self.caseno, self.permno) 165 else: 166 return '%s#%d' % ( 167 self.suite.name, self.caseno) 168 169 def permute(self, class_=None, defines={}, permno=None, **_): 170 ncase = (class_ or type(self))(self.config) 171 for k, v in self.__dict__.items(): 172 setattr(ncase, k, v) 173 ncase.case = self 174 ncase.perms = [ncase] 175 ncase.permno = permno 176 ncase.defines = defines 177 return ncase 178 179 def build(self, f, **_): 180 # prologue 181 for k, v in sorted(self.defines.items()): 182 if k not in self.suite.defines: 183 f.write('#define %s %s\n' % (k, v)) 184 185 f.write('void test_case%d(%s) {' % (self.caseno, ','.join( 186 '\n'+8*' '+'__attribute__((unused)) intmax_t %s' % k 187 for k in sorted(self.perms[0].defines) 188 if k not in self.defines))) 189 190 f.write(PROLOGUE) 191 f.write('\n') 192 f.write(4*' '+'// test case %d\n' % self.caseno) 193 f.write(4*' '+'#line %d "%s"\n' % (self.code_lineno, self.suite.path)) 194 195 # test case goes here 196 f.write(self.code) 197 198 # epilogue 199 f.write(EPILOGUE) 200 f.write('}\n') 201 202 for k, v in sorted(self.defines.items()): 203 if k not in self.suite.defines: 204 f.write('#undef %s\n' % k) 205 206 def shouldtest(self, **args): 207 if (self.filter is not None and 208 len(self.filter) >= 1 and 209 self.filter[0] != self.caseno): 210 return False 211 elif (self.filter is not None and 212 len(self.filter) >= 2 and 213 self.filter[1] != self.permno): 214 return False 215 elif args.get('no_internal') and self.in_ is not None: 216 return False 217 elif self.if_ is not None: 218 if_ = self.if_ 219 while True: 220 for k, v in sorted(self.defines.items(), 221 key=lambda x: len(x[0]), reverse=True): 222 if k in if_: 223 if_ = if_.replace(k, '(%s)' % v) 224 break 225 else: 226 break 227 if_ = ( 228 re.sub('(\&\&|\?)', ' and ', 229 re.sub('(\|\||:)', ' or ', 230 re.sub('!(?!=)', ' not ', if_)))) 231 return eval(if_) 232 else: 233 return True 234 235 def test(self, exec=[], persist=False, cycles=None, 236 gdb=False, failure=None, disk=None, **args): 237 # build command 238 cmd = exec + ['./%s.test' % self.suite.path, 239 repr(self.caseno), repr(self.permno)] 240 241 # persist disk or keep in RAM for speed? 242 if persist: 243 if not disk: 244 disk = self.suite.path + '.disk' 245 if persist != 'noerase': 246 try: 247 with open(disk, 'w') as f: 248 f.truncate(0) 249 if args.get('verbose'): 250 print('truncate --size=0', disk) 251 except FileNotFoundError: 252 pass 253 254 cmd.append(disk) 255 256 # simulate power-loss after n cycles? 257 if cycles: 258 cmd.append(str(cycles)) 259 260 # failed? drop into debugger? 261 if gdb and failure: 262 ncmd = ['gdb'] 263 if gdb == 'assert': 264 ncmd.extend(['-ex', 'r']) 265 if failure.assert_: 266 ncmd.extend(['-ex', 'up 2']) 267 elif gdb == 'main': 268 ncmd.extend([ 269 '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno), 270 '-ex', 'r']) 271 ncmd.extend(['--args'] + cmd) 272 273 if args.get('verbose'): 274 print(' '.join(shlex.quote(c) for c in ncmd)) 275 signal.signal(signal.SIGINT, signal.SIG_IGN) 276 sys.exit(sp.call(ncmd)) 277 278 # run test case! 279 mpty, spty = pty.openpty() 280 if args.get('verbose'): 281 print(' '.join(shlex.quote(c) for c in cmd)) 282 proc = sp.Popen(cmd, stdout=spty, stderr=spty) 283 os.close(spty) 284 mpty = os.fdopen(mpty, 'r', 1) 285 stdout = [] 286 assert_ = None 287 try: 288 while True: 289 try: 290 line = mpty.readline() 291 except OSError as e: 292 if e.errno == errno.EIO: 293 break 294 raise 295 stdout.append(line) 296 if args.get('verbose'): 297 sys.stdout.write(line) 298 # intercept asserts 299 m = re.match( 300 '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' 301 .format('(?:\033\[[\d;]*.| )*', 'assert'), 302 line) 303 if m and assert_ is None: 304 try: 305 with open(m.group(1)) as f: 306 lineno = int(m.group(2)) 307 line = (next(it.islice(f, lineno-1, None)) 308 .strip('\n')) 309 assert_ = { 310 'path': m.group(1), 311 'line': line, 312 'lineno': lineno, 313 'message': m.group(3)} 314 except: 315 pass 316 except KeyboardInterrupt: 317 raise TestFailure(self, 1, stdout, None) 318 proc.wait() 319 320 # did we pass? 321 if proc.returncode != 0: 322 raise TestFailure(self, proc.returncode, stdout, assert_) 323 else: 324 return PASS 325 326class ValgrindTestCase(TestCase): 327 def __init__(self, config, **args): 328 self.leaky = config.get('leaky', False) 329 super().__init__(config, **args) 330 331 def shouldtest(self, **args): 332 return not self.leaky and super().shouldtest(**args) 333 334 def test(self, exec=[], **args): 335 verbose = args.get('verbose') 336 uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1) 337 exec = [ 338 'valgrind', 339 '--leak-check=full', 340 ] + (['--undef-value-errors=no'] if uninit else []) + [ 341 ] + (['--track-origins=yes'] if not uninit else []) + [ 342 '--error-exitcode=4', 343 '--error-limit=no', 344 ] + (['--num-callers=1'] if not verbose else []) + [ 345 '-q'] + exec 346 return super().test(exec=exec, **args) 347 348class ReentrantTestCase(TestCase): 349 def __init__(self, config, **args): 350 self.reentrant = config.get('reentrant', False) 351 super().__init__(config, **args) 352 353 def shouldtest(self, **args): 354 return self.reentrant and super().shouldtest(**args) 355 356 def test(self, persist=False, gdb=False, failure=None, **args): 357 for cycles in it.count(1): 358 # clear disk first? 359 if cycles == 1 and persist != 'noerase': 360 persist = 'erase' 361 else: 362 persist = 'noerase' 363 364 # exact cycle we should drop into debugger? 365 if gdb and failure and failure.cycleno == cycles: 366 return super().test(gdb=gdb, persist=persist, cycles=cycles, 367 failure=failure, **args) 368 369 # run tests, but kill the program after prog/erase has 370 # been hit n cycles. We exit with a special return code if the 371 # program has not finished, since this isn't a test failure. 372 try: 373 return super().test(persist=persist, cycles=cycles, **args) 374 except TestFailure as nfailure: 375 if nfailure.returncode == 33: 376 continue 377 else: 378 nfailure.cycleno = cycles 379 raise 380 381class TestSuite: 382 def __init__(self, path, classes=[TestCase], defines={}, 383 filter=None, **args): 384 self.name = os.path.basename(path) 385 if self.name.endswith('.toml'): 386 self.name = self.name[:-len('.toml')] 387 if args.get('build_dir'): 388 self.toml = path 389 self.path = args['build_dir'] + '/' + path 390 else: 391 self.toml = path 392 self.path = path 393 self.classes = classes 394 self.defines = defines.copy() 395 self.filter = filter 396 397 with open(self.toml) as f: 398 # load tests 399 config = toml.load(f) 400 401 # find line numbers 402 f.seek(0) 403 linenos = [] 404 code_linenos = [] 405 for i, line in enumerate(f): 406 if re.match(r'\[\[\s*case\s*\]\]', line): 407 linenos.append(i+1) 408 if re.match(r'code\s*=\s*(\'\'\'|""")', line): 409 code_linenos.append(i+2) 410 411 code_linenos.reverse() 412 413 # grab global config 414 for k, v in config.get('define', {}).items(): 415 if k not in self.defines: 416 self.defines[k] = v 417 self.code = config.get('code', None) 418 if self.code is not None: 419 self.code_lineno = code_linenos.pop() 420 421 # create initial test cases 422 self.cases = [] 423 for i, (case, lineno) in enumerate(zip(config['case'], linenos)): 424 # code lineno? 425 if 'code' in case: 426 case['code_lineno'] = code_linenos.pop() 427 # merge conditions if necessary 428 if 'if' in config and 'if' in case: 429 case['if'] = '(%s) && (%s)' % (config['if'], case['if']) 430 elif 'if' in config: 431 case['if'] = config['if'] 432 # initialize test case 433 self.cases.append(TestCase(case, filter=filter, 434 suite=self, caseno=i+1, lineno=lineno, **args)) 435 436 def __str__(self): 437 return self.name 438 439 def __lt__(self, other): 440 return self.name < other.name 441 442 def permute(self, **args): 443 for case in self.cases: 444 # lets find all parameterized definitions, in one of [args.D, 445 # suite.defines, case.defines, DEFINES]. Note that each of these 446 # can be either a dict of defines, or a list of dicts, expressing 447 # an initial set of permutations. 448 pending = [{}] 449 for inits in [self.defines, case.defines, DEFINES]: 450 if not isinstance(inits, list): 451 inits = [inits] 452 453 npending = [] 454 for init, pinit in it.product(inits, pending): 455 ninit = pinit.copy() 456 for k, v in init.items(): 457 if k not in ninit: 458 try: 459 ninit[k] = eval(v) 460 except: 461 ninit[k] = v 462 npending.append(ninit) 463 464 pending = npending 465 466 # expand permutations 467 pending = list(reversed(pending)) 468 expanded = [] 469 while pending: 470 perm = pending.pop() 471 for k, v in sorted(perm.items()): 472 if not isinstance(v, str) and isinstance(v, abc.Iterable): 473 for nv in reversed(v): 474 nperm = perm.copy() 475 nperm[k] = nv 476 pending.append(nperm) 477 break 478 else: 479 expanded.append(perm) 480 481 # generate permutations 482 case.perms = [] 483 for i, (class_, defines) in enumerate( 484 it.product(self.classes, expanded)): 485 case.perms.append(case.permute( 486 class_, defines, permno=i+1, **args)) 487 488 # also track non-unique defines 489 case.defines = {} 490 for k, v in case.perms[0].defines.items(): 491 if all(perm.defines[k] == v for perm in case.perms): 492 case.defines[k] = v 493 494 # track all perms and non-unique defines 495 self.perms = [] 496 for case in self.cases: 497 self.perms.extend(case.perms) 498 499 self.defines = {} 500 for k, v in self.perms[0].defines.items(): 501 if all(perm.defines.get(k, None) == v for perm in self.perms): 502 self.defines[k] = v 503 504 return self.perms 505 506 def build(self, **args): 507 # build test files 508 tf = open(self.path + '.test.tc', 'w') 509 tf.write(GLOBALS) 510 if self.code is not None: 511 tf.write('#line %d "%s"\n' % (self.code_lineno, self.path)) 512 tf.write(self.code) 513 514 tfs = {None: tf} 515 for case in self.cases: 516 if case.in_ not in tfs: 517 tfs[case.in_] = open(self.path+'.'+ 518 re.sub('(\.c)?$', '.tc', case.in_.replace('/', '.')), 'w') 519 tfs[case.in_].write('#line 1 "%s"\n' % case.in_) 520 with open(case.in_) as f: 521 for line in f: 522 tfs[case.in_].write(line) 523 tfs[case.in_].write('\n') 524 tfs[case.in_].write(GLOBALS) 525 526 tfs[case.in_].write('\n') 527 case.build(tfs[case.in_], **args) 528 529 tf.write('\n') 530 tf.write('const char *lfs_testbd_path;\n') 531 tf.write('uint32_t lfs_testbd_cycles;\n') 532 tf.write('int main(int argc, char **argv) {\n') 533 tf.write(4*' '+'int case_ = (argc > 1) ? atoi(argv[1]) : 0;\n') 534 tf.write(4*' '+'int perm = (argc > 2) ? atoi(argv[2]) : 0;\n') 535 tf.write(4*' '+'lfs_testbd_path = (argc > 3) ? argv[3] : NULL;\n') 536 tf.write(4*' '+'lfs_testbd_cycles = (argc > 4) ? atoi(argv[4]) : 0;\n') 537 for perm in self.perms: 538 # test declaration 539 tf.write(4*' '+'extern void test_case%d(%s);\n' % ( 540 perm.caseno, ', '.join( 541 'intmax_t %s' % k for k in sorted(perm.defines) 542 if k not in perm.case.defines))) 543 # test call 544 tf.write(4*' '+ 545 'if (argc < 3 || (case_ == %d && perm == %d)) {' 546 ' test_case%d(%s); ' 547 '}\n' % (perm.caseno, perm.permno, perm.caseno, ', '.join( 548 str(v) for k, v in sorted(perm.defines.items()) 549 if k not in perm.case.defines))) 550 tf.write('}\n') 551 552 for tf in tfs.values(): 553 tf.close() 554 555 # write makefiles 556 with open(self.path + '.mk', 'w') as mk: 557 mk.write(RULES.replace(4*' ', '\t') % dict(path=self.path)) 558 mk.write('\n') 559 560 # add coverage hooks? 561 if args.get('coverage'): 562 mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict( 563 path=self.path)) 564 mk.write('\n') 565 566 # add truely global defines globally 567 for k, v in sorted(self.defines.items()): 568 mk.write('%s.test: override CFLAGS += -D%s=%r\n' 569 % (self.path, k, v)) 570 571 for path in tfs: 572 if path is None: 573 mk.write('%s: %s | %s\n' % ( 574 self.path+'.test.c', 575 self.toml, 576 self.path+'.test.tc')) 577 else: 578 mk.write('%s: %s %s | %s\n' % ( 579 self.path+'.'+path.replace('/', '.'), 580 self.toml, 581 path, 582 self.path+'.'+re.sub('(\.c)?$', '.tc', 583 path.replace('/', '.')))) 584 mk.write('\t./scripts/explode_asserts.py $| -o $@\n') 585 586 self.makefile = self.path + '.mk' 587 self.target = self.path + '.test' 588 return self.makefile, self.target 589 590 def test(self, **args): 591 # run test suite! 592 if not args.get('verbose', True): 593 sys.stdout.write(self.name + ' ') 594 sys.stdout.flush() 595 for perm in self.perms: 596 if not perm.shouldtest(**args): 597 continue 598 599 try: 600 result = perm.test(**args) 601 except TestFailure as failure: 602 perm.result = failure 603 if not args.get('verbose', True): 604 sys.stdout.write(FAIL) 605 sys.stdout.flush() 606 if not args.get('keep_going'): 607 if not args.get('verbose', True): 608 sys.stdout.write('\n') 609 raise 610 else: 611 perm.result = PASS 612 if not args.get('verbose', True): 613 sys.stdout.write(PASS) 614 sys.stdout.flush() 615 616 if not args.get('verbose', True): 617 sys.stdout.write('\n') 618 619def main(**args): 620 # figure out explicit defines 621 defines = {} 622 for define in args['D']: 623 k, v, *_ = define.split('=', 2) + [''] 624 defines[k] = v 625 626 # and what class of TestCase to run 627 classes = [] 628 if args.get('normal'): 629 classes.append(TestCase) 630 if args.get('reentrant'): 631 classes.append(ReentrantTestCase) 632 if args.get('valgrind'): 633 classes.append(ValgrindTestCase) 634 if not classes: 635 classes = [TestCase] 636 637 suites = [] 638 for testpath in args['test_paths']: 639 # optionally specified test case/perm 640 testpath, *filter = testpath.split('#') 641 filter = [int(f) for f in filter] 642 643 # figure out the suite's toml file 644 if os.path.isdir(testpath): 645 testpath = testpath + '/*.toml' 646 elif os.path.isfile(testpath): 647 testpath = testpath 648 elif testpath.endswith('.toml'): 649 testpath = TEST_PATHS + '/' + testpath 650 else: 651 testpath = TEST_PATHS + '/' + testpath + '.toml' 652 653 # find tests 654 for path in glob.glob(testpath): 655 suites.append(TestSuite(path, classes, defines, filter, **args)) 656 657 # sort for reproducability 658 suites = sorted(suites) 659 660 # generate permutations 661 for suite in suites: 662 suite.permute(**args) 663 664 # build tests in parallel 665 print('====== building ======') 666 makefiles = [] 667 targets = [] 668 for suite in suites: 669 makefile, target = suite.build(**args) 670 makefiles.append(makefile) 671 targets.append(target) 672 673 cmd = (['make', '-f', 'Makefile'] + 674 list(it.chain.from_iterable(['-f', m] for m in makefiles)) + 675 [target for target in targets]) 676 mpty, spty = pty.openpty() 677 if args.get('verbose'): 678 print(' '.join(shlex.quote(c) for c in cmd)) 679 proc = sp.Popen(cmd, stdout=spty, stderr=spty) 680 os.close(spty) 681 mpty = os.fdopen(mpty, 'r', 1) 682 stdout = [] 683 while True: 684 try: 685 line = mpty.readline() 686 except OSError as e: 687 if e.errno == errno.EIO: 688 break 689 raise 690 stdout.append(line) 691 if args.get('verbose'): 692 sys.stdout.write(line) 693 # intercept warnings 694 m = re.match( 695 '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$' 696 .format('(?:\033\[[\d;]*.| )*', 'warning'), 697 line) 698 if m and not args.get('verbose'): 699 try: 700 with open(m.group(1)) as f: 701 lineno = int(m.group(2)) 702 line = next(it.islice(f, lineno-1, None)).strip('\n') 703 sys.stdout.write( 704 "\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m " 705 "{message}\n{line}\n\n".format( 706 path=m.group(1), line=line, lineno=lineno, 707 message=m.group(3))) 708 except: 709 pass 710 proc.wait() 711 if proc.returncode != 0: 712 if not args.get('verbose'): 713 for line in stdout: 714 sys.stdout.write(line) 715 sys.exit(-1) 716 717 print('built %d test suites, %d test cases, %d permutations' % ( 718 len(suites), 719 sum(len(suite.cases) for suite in suites), 720 sum(len(suite.perms) for suite in suites))) 721 722 total = 0 723 for suite in suites: 724 for perm in suite.perms: 725 total += perm.shouldtest(**args) 726 if total != sum(len(suite.perms) for suite in suites): 727 print('filtered down to %d permutations' % total) 728 729 # only requested to build? 730 if args.get('build'): 731 return 0 732 733 print('====== testing ======') 734 try: 735 for suite in suites: 736 suite.test(**args) 737 except TestFailure: 738 pass 739 740 print('====== results ======') 741 passed = 0 742 failed = 0 743 for suite in suites: 744 for perm in suite.perms: 745 if perm.result == PASS: 746 passed += 1 747 elif isinstance(perm.result, TestFailure): 748 sys.stdout.write( 749 "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m " 750 "{perm} failed\n".format( 751 perm=perm, path=perm.suite.path, lineno=perm.lineno, 752 returncode=perm.result.returncode or 0)) 753 if perm.result.stdout: 754 if perm.result.assert_: 755 stdout = perm.result.stdout[:-1] 756 else: 757 stdout = perm.result.stdout 758 for line in stdout[-5:]: 759 sys.stdout.write(line) 760 if perm.result.assert_: 761 sys.stdout.write( 762 "\033[01m{path}:{lineno}:\033[01;31massert:\033[m " 763 "{message}\n{line}\n".format( 764 **perm.result.assert_)) 765 sys.stdout.write('\n') 766 failed += 1 767 768 if args.get('coverage'): 769 # collect coverage info 770 # why -j1? lcov doesn't work in parallel because of gcov limitations 771 cmd = (['make', '-j1', '-f', 'Makefile'] + 772 list(it.chain.from_iterable(['-f', m] for m in makefiles)) + 773 (['COVERAGETARGET=%s' % args['coverage']] 774 if isinstance(args['coverage'], str) else []) + 775 [suite.path + '.info' for suite in suites 776 if any(perm.result == PASS for perm in suite.perms)]) 777 if args.get('verbose'): 778 print(' '.join(shlex.quote(c) for c in cmd)) 779 proc = sp.Popen(cmd, 780 stdout=sp.PIPE if not args.get('verbose') else None, 781 stderr=sp.STDOUT if not args.get('verbose') else None, 782 universal_newlines=True) 783 proc.wait() 784 if proc.returncode != 0: 785 if not args.get('verbose'): 786 for line in proc.stdout: 787 sys.stdout.write(line) 788 sys.exit(-1) 789 790 if args.get('gdb'): 791 failure = None 792 for suite in suites: 793 for perm in suite.perms: 794 if isinstance(perm.result, TestFailure): 795 failure = perm.result 796 if failure is not None: 797 print('======= gdb ======') 798 # drop into gdb 799 failure.case.test(failure=failure, **args) 800 sys.exit(0) 801 802 print('tests passed %d/%d (%.2f%%)' % (passed, total, 803 100*(passed/total if total else 1.0))) 804 print('tests failed %d/%d (%.2f%%)' % (failed, total, 805 100*(failed/total if total else 1.0))) 806 return 1 if failed > 0 else 0 807 808if __name__ == "__main__": 809 import argparse 810 parser = argparse.ArgumentParser( 811 description="Run parameterized tests in various configurations.") 812 parser.add_argument('test_paths', nargs='*', default=[TEST_PATHS], 813 help="Description of test(s) to run. By default, this is all tests \ 814 found in the \"{0}\" directory. Here, you can specify a different \ 815 directory of tests, a specific file, a suite by name, and even \ 816 specific test cases and permutations. For example \ 817 \"test_dirs#1\" or \"{0}/test_dirs.toml#1#1\".".format(TEST_PATHS)) 818 parser.add_argument('-D', action='append', default=[], 819 help="Overriding parameter definitions.") 820 parser.add_argument('-v', '--verbose', action='store_true', 821 help="Output everything that is happening.") 822 parser.add_argument('-k', '--keep-going', action='store_true', 823 help="Run all tests instead of stopping on first error. Useful for CI.") 824 parser.add_argument('-p', '--persist', choices=['erase', 'noerase'], 825 nargs='?', const='erase', 826 help="Store disk image in a file.") 827 parser.add_argument('-b', '--build', action='store_true', 828 help="Only build the tests, do not execute.") 829 parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'], 830 nargs='?', const='assert', 831 help="Drop into gdb on test failure.") 832 parser.add_argument('--no-internal', action='store_true', 833 help="Don't run tests that require internal knowledge.") 834 parser.add_argument('-n', '--normal', action='store_true', 835 help="Run tests normally.") 836 parser.add_argument('-r', '--reentrant', action='store_true', 837 help="Run reentrant tests with simulated power-loss.") 838 parser.add_argument('--valgrind', action='store_true', 839 help="Run non-leaky tests under valgrind to check for memory leaks.") 840 parser.add_argument('--exec', default=[], type=lambda e: e.split(), 841 help="Run tests with another executable prefixed on the command line.") 842 parser.add_argument('--disk', 843 help="Specify a file to use for persistent/reentrant tests.") 844 parser.add_argument('--coverage', type=lambda x: x if x else True, 845 nargs='?', const='', 846 help="Collect coverage information during testing. This uses lcov/gcov \ 847 to accumulate coverage information into *.info files. May also \ 848 a path to a *.info file to accumulate coverage info into.") 849 parser.add_argument('--build-dir', 850 help="Build relative to the specified directory instead of the \ 851 current directory.") 852 853 sys.exit(main(**vars(parser.parse_args()))) 854