Lines Matching +full:check +full:- +full:test +full:- +full:suites
6 # ./scripts/test.py runners/test_runner -b
9 # SPDX-License-Identifier: BSD-3-Clause
38 def openio(path, mode='r', buffering=-1):
39 # allow '-' for stdin/stdout
40 if path == '-':
89 parens -= 1
100 # which can be comma-separated values, with optional
153 self.name = self.name[:-len('.toml')]
155 # load toml file and parse test cases
216 # combine per-case defines
220 # combine other per-case things
244 print('no test suites found in %r?' % test_paths)
245 sys.exit(-1)
247 # load the suites
248 suites = [TestSuite(path, args) for path in paths]
249 suites.sort(key=lambda s: s.name)
251 # check for name conflicts, these will cause ambiguity problems later
254 for suite in suites:
279 # we can only compile one test suite at a time
281 if len(suites) > 1:
282 print('more than one test suite for compilation? (%r)' % test_paths)
283 sys.exit(-1)
285 suite = suites[0]
287 # write generated test source
316 # the test defines
342 f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
363 f.writeln(4*' '+'// test case %s' % case.name)
388 f.writeln('#define %-24s '
390 f.writeln('#define %-24s '
418 # 32-bytes for some reason
435 f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
474 for suite in suites:
485 f.writeln('#define %-24s '
488 f.writeln('#define %-24s '
520 '--leak-check=full',
521 '--track-origins=yes',
522 '--error-exitcode=4',
523 '-q']
528 '-R',
529 '--perf-freq=%s' % args['perf_freq']
531 '--perf-period=%s' % args['perf_period']
533 '--perf-events=%s' % args['perf_events']
535 '--perf-path=%s' % args['perf_path']
537 '-o%s' % args['perf']]))
541 cmd.append('-G%s' % args['geometry'])
543 cmd.append('-P%s' % args['powerloss'])
545 cmd.append('-d%s' % args['disk'])
547 cmd.append('-t%s' % args['trace'])
549 cmd.append('--trace-backtrace')
551 cmd.append('--trace-period=%s' % args['trace_period'])
553 cmd.append('--trace-freq=%s' % args['trace_freq'])
555 cmd.append('--read-sleep=%s' % args['read_sleep'])
557 cmd.append('--prog-sleep=%s' % args['prog_sleep'])
559 cmd.append('--erase-sleep=%s' % args['erase_sleep'])
564 cmd.append('-D%s' % define)
570 if args.get('summary'): cmd.append('--summary')
571 if args.get('list_suites'): cmd.append('--list-suites')
572 if args.get('list_cases'): cmd.append('--list-cases')
573 if args.get('list_suite_paths'): cmd.append('--list-suite-paths')
574 if args.get('list_case_paths'): cmd.append('--list-case-paths')
575 if args.get('list_defines'): cmd.append('--list-defines')
577 cmd.append('--list-permutation-defines')
579 cmd.append('--list-implicit-defines')
580 if args.get('list_geometries'): cmd.append('--list-geometries')
581 if args.get('list_powerlosses'): cmd.append('--list-powerlosses')
595 cmd = runner_ + ['--list-cases'] + ids
622 sys.exit(-1)
625 cmd = runner_ + ['--list-case-paths'] + ids
645 suite = suite[:-len('.toml')]
652 sys.exit(-1)
669 cmd = runner_ + ['--list-case-paths', id]
693 sys.exit(-1)
699 cmd = runner_ + ['--list-permutation-defines', id]
721 sys.exit(-1)
726 # Thread-safe CSV writer
753 self.head.extend(row.keys() - (self.head + self.tail))
761 # A test failure
893 job_runner.append('-s%s,%s,%s' % (start, start+step, step))
895 job_runner.append('-s%s,,%s' % (start, step))
924 # resume after failed test
958 '%d/%d suites' % (
1015 print('found %d suites, %d cases, %d/%d permutations' % (
1097 'in %.2fs' % (stop-start)]))))
1123 stdout = stdout[:-1]
1124 for line in stdout[-args.get('context', 5):]:
1136 line = next(it.islice(f, lineno-1, None)).strip('\n')
1155 '-ex', 'break main',
1156 '-ex', 'break %s:%d' % (path, lineno),
1157 '-ex', 'run',
1158 '--args']
1162 '-ex', 'break %s:%d' % (path, lineno),
1163 '-ex', 'run',
1164 '--args']
1168 '-ex', 'break %s:%d' % (path, lineno),
1169 '-ex', 'ignore 1 %d' % args['gdb_pl'],
1170 '-ex', 'run',
1171 '--args']
1175 sum(1 for _ in re.finditer('[0-9a-f]',
1176 failure.id.split(':', 2)[-1]))
1180 '-ex', 'break %s:%d' % (path, lineno),
1181 '-ex', 'ignore 1 %d' % max(powerlosses-1, 0),
1182 '-ex', 'run',
1183 '--args']
1187 sum(1 for _ in re.finditer('[0-9a-f]',
1188 failure.id.split(':', 2)[-1]))
1192 '-ex', 'break %s:%d' % (path, lineno),
1193 '-ex', 'ignore 1 %d' % powerlosses,
1194 '-ex', 'run',
1195 '--args']
1198 '-ex', 'run',
1199 '-ex', 'frame function raise',
1200 '-ex', 'up 2',
1201 '--args']
1204 '-ex', 'run',
1205 '--args']
1251 '-v', '--verbose',
1255 '--color',
1260 # test flags
1261 test_parser = parser.add_argument_group('test options')
1266 help="Test runner to use for testing. Defaults to %r." % RUNNER_PATH)
1272 '-Y', '--summary',
1276 '-l', '--list-suites',
1278 help="List test suites.")
1280 '-L', '--list-cases',
1282 help="List test cases.")
1284 '--list-suite-paths',
1286 help="List the path for each test suite.")
1288 '--list-case-paths',
1290 help="List the path and line number for each test case.")
1292 '--list-defines',
1294 help="List all defines in this test-runner.")
1296 '--list-permutation-defines',
1298 help="List explicit defines in this test-runner.")
1300 '--list-implicit-defines',
1302 help="List implicit defines in this test-runner.")
1304 '--list-geometries',
1308 '--list-powerlosses',
1310 help="List the available power-loss scenarios.")
1312 '-D', '--define',
1314 help="Override a test define.")
1316 '-G', '--geometry',
1317 help="Comma-separated list of disk geometries to test.")
1319 '-P', '--powerloss',
1320 help="Comma-separated list of power-loss scenarios to test.")
1322 '-d', '--disk',
1325 '-t', '--trace',
1328 '--trace-backtrace',
1332 '--trace-period',
1335 '--trace-freq',
1338 '-O', '--stdout',
1341 '-o', '--output',
1344 '--read-sleep',
1347 '--prog-sleep',
1350 '--erase-sleep',
1353 '-j', '--jobs',
1359 '-k', '--keep-going',
1363 '-i', '--isolate',
1365 help="Run each test permutation in a separate process.")
1367 '-b', '--by-suites',
1371 '-B', '--by-cases',
1375 '--context',
1378 help="Show this many lines of stdout on test failure. "
1381 '--gdb',
1383 help="Drop into gdb on test failure.")
1385 '--gdb-case',
1387 help="Drop into gdb on test failure but stop at the beginning "
1388 "of the failing test case.")
1390 '--gdb-main',
1392 help="Drop into gdb on test failure but stop at the beginning "
1395 '--gdb-pl',
1399 '--gdb-pl-before',
1403 '--gdb-pl-after',
1407 '--gdb-path',
1413 '--exec',
1417 '--valgrind',
1420 "--isolate.")
1422 '--valgrind-path',
1428 '-p', '--perf',
1432 '--perf-freq',
1436 '--perf-period',
1440 '--perf-events',
1444 '--perf-script',
1449 '--perf-path',
1462 '-c', '--compile',
1464 help="Compile a test suite or source file.")
1466 '-s', '--source',
1469 '--include',
1471 help="Inject this header file into every compiled test file. "
1474 '-o', '--output',