1import argparse 2import os 3import sys 4from test import support 5 6 7USAGE = """\ 8python -m test [options] [test_name1 [test_name2 ...]] 9python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] 10""" 11 12DESCRIPTION = """\ 13Run Python regression tests. 14 15If no arguments or options are provided, finds all files matching 16the pattern "test_*" in the Lib/test subdirectory and runs 17them in alphabetical order (but see -M and -u, below, for exceptions). 18 19For more rigorous testing, it is useful to use the following 20command line: 21 22python -E -Wd -m test [options] [test_name1 ...] 23""" 24 25EPILOG = """\ 26Additional option details: 27 28-r randomizes test execution order. You can use --randseed=int to provide an 29int seed value for the randomizer; this is useful for reproducing troublesome 30test orders. 31 32-s On the first invocation of regrtest using -s, the first test file found 33or the first test file given on the command line is run, and the name of 34the next test is recorded in a file named pynexttest. If run from the 35Python build directory, pynexttest is located in the 'build' subdirectory, 36otherwise it is located in tempfile.gettempdir(). On subsequent runs, 37the test in pynexttest is run, and the next test is written to pynexttest. 38When the last test has been run, pynexttest is deleted. In this way it 39is possible to single step through the test files. This is useful when 40doing memory analysis on the Python interpreter, which process tends to 41consume too many resources to run the full regression test non-stop. 42 43-S is used to continue running tests after an aborted run. It will 44maintain the order a standard run (ie, this assumes -r is not used). 45This is useful after the tests have prematurely stopped for some external 46reason and you want to start running from where you left off rather 47than starting from the beginning. 48 49-f reads the names of tests from the file given as f's argument, one 50or more test names per line. Whitespace is ignored. Blank lines and 51lines beginning with '#' are ignored. This is especially useful for 52whittling down failures involving interactions among tests. 53 54-L causes the leaks(1) command to be run just before exit if it exists. 55leaks(1) is available on Mac OS X and presumably on some other 56FreeBSD-derived systems. 57 58-R runs each test several times and examines sys.gettotalrefcount() to 59see if the test appears to be leaking references. The argument should 60be of the form stab:run:fname where 'stab' is the number of times the 61test is run to let gettotalrefcount settle down, 'run' is the number 62of times further it is run and 'fname' is the name of the file the 63reports are written to. These parameters all have defaults (5, 4 and 64"reflog.txt" respectively), and the minimal invocation is '-R :'. 65 66-M runs tests that require an exorbitant amount of memory. These tests 67typically try to ascertain containers keep working when containing more than 682 billion objects, which only works on 64-bit systems. There are also some 69tests that try to exhaust the address space of the process, which only makes 70sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, 71which is a string in the form of '2.5Gb', determines how much memory the 72tests will limit themselves to (but they may go slightly over.) The number 73shouldn't be more memory than the machine has (including swap memory). You 74should also keep in mind that swap memory is generally much, much slower 75than RAM, and setting memlimit to all available RAM or higher will heavily 76tax the machine. On the other hand, it is no use running these tests with a 77limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect 78to use more than memlimit memory will be skipped. The big-memory tests 79generally run very, very long. 80 81-u is used to specify which special resource intensive tests to run, 82such as those requiring large file support or network connectivity. 83The argument is a comma-separated list of words indicating the 84resources to test. Currently only the following are defined: 85 86 all - Enable all special resources. 87 88 none - Disable all special resources (this is the default). 89 90 audio - Tests that use the audio device. (There are known 91 cases of broken audio drivers that can crash Python or 92 even the Linux kernel.) 93 94 curses - Tests that use curses and will modify the terminal's 95 state and output modes. 96 97 largefile - It is okay to run some test that may create huge 98 files. These tests can take a long time and may 99 consume >2 GiB of disk space temporarily. 100 101 network - It is okay to run tests that use external network 102 resource, e.g. testing SSL support for sockets. 103 104 decimal - Test the decimal module against a large suite that 105 verifies compliance with standards. 106 107 cpu - Used for certain CPU-heavy tests. 108 109 subprocess Run all tests for the subprocess module. 110 111 urlfetch - It is okay to download files required on testing. 112 113 gui - Run tests that require a running GUI. 114 115 tzdata - Run tests that require timezone data. 116 117To enable all resources except one, use '-uall,-<resource>'. For 118example, to run all the tests except for the gui tests, give the 119option '-uall,-gui'. 120 121--matchfile filters tests using a text file, one pattern per line. 122Pattern examples: 123 124- test method: test_stat_attributes 125- test class: FileTests 126- test identifier: test_os.FileTests.test_stat_attributes 127""" 128 129 130ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', 131 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') 132 133# Other resources excluded from --use=all: 134# 135# - extralagefile (ex: test_zipfile64): really too slow to be enabled 136# "by default" 137# - tzdata: while needed to validate fully test_datetime, it makes 138# test_datetime too slow (15-20 min on some buildbots) and so is disabled by 139# default (see bpo-30822). 140RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') 141 142class _ArgParser(argparse.ArgumentParser): 143 144 def error(self, message): 145 super().error(message + "\nPass -h or --help for complete help.") 146 147 148def _create_parser(): 149 # Set prog to prevent the uninformative "__main__.py" from displaying in 150 # error messages when using "python -m test ...". 151 parser = _ArgParser(prog='regrtest.py', 152 usage=USAGE, 153 description=DESCRIPTION, 154 epilog=EPILOG, 155 add_help=False, 156 formatter_class=argparse.RawDescriptionHelpFormatter) 157 158 # Arguments with this clause added to its help are described further in 159 # the epilog's "Additional option details" section. 160 more_details = ' See the section at bottom for more details.' 161 162 group = parser.add_argument_group('General options') 163 # We add help explicitly to control what argument group it renders under. 164 group.add_argument('-h', '--help', action='help', 165 help='show this help message and exit') 166 group.add_argument('--timeout', metavar='TIMEOUT', type=float, 167 help='dump the traceback and exit if a test takes ' 168 'more than TIMEOUT seconds; disabled if TIMEOUT ' 169 'is negative or equals to zero') 170 group.add_argument('--wait', action='store_true', 171 help='wait for user input, e.g., allow a debugger ' 172 'to be attached') 173 group.add_argument('--worker-args', metavar='ARGS') 174 group.add_argument('-S', '--start', metavar='START', 175 help='the name of the test at which to start.' + 176 more_details) 177 178 group = parser.add_argument_group('Verbosity') 179 group.add_argument('-v', '--verbose', action='count', 180 help='run tests in verbose mode with output to stdout') 181 group.add_argument('-w', '--verbose2', action='store_true', 182 help='re-run failed tests in verbose mode') 183 group.add_argument('-W', '--verbose3', action='store_true', 184 help='display test output on failure') 185 group.add_argument('-q', '--quiet', action='store_true', 186 help='no output unless one or more tests fail') 187 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow', 188 help='print the slowest 10 tests') 189 group.add_argument('--header', action='store_true', 190 help='print header with interpreter info') 191 192 group = parser.add_argument_group('Selecting tests') 193 group.add_argument('-r', '--randomize', action='store_true', 194 help='randomize test execution order.' + more_details) 195 group.add_argument('--randseed', metavar='SEED', 196 dest='random_seed', type=int, 197 help='pass a random seed to reproduce a previous ' 198 'random run') 199 group.add_argument('-f', '--fromfile', metavar='FILE', 200 help='read names of tests to run from a file.' + 201 more_details) 202 group.add_argument('-x', '--exclude', action='store_true', 203 help='arguments are tests to *exclude*') 204 group.add_argument('-s', '--single', action='store_true', 205 help='single step through a set of tests.' + 206 more_details) 207 group.add_argument('-m', '--match', metavar='PAT', 208 dest='match_tests', action='append', 209 help='match test cases and methods with glob pattern PAT') 210 group.add_argument('-i', '--ignore', metavar='PAT', 211 dest='ignore_tests', action='append', 212 help='ignore test cases and methods with glob pattern PAT') 213 group.add_argument('--matchfile', metavar='FILENAME', 214 dest='match_filename', 215 help='similar to --match but get patterns from a ' 216 'text file, one pattern per line') 217 group.add_argument('--ignorefile', metavar='FILENAME', 218 dest='ignore_filename', 219 help='similar to --matchfile but it receives patterns ' 220 'from text file to ignore') 221 group.add_argument('-G', '--failfast', action='store_true', 222 help='fail as soon as a test fails (only with -v or -W)') 223 group.add_argument('-u', '--use', metavar='RES1,RES2,...', 224 action='append', type=resources_list, 225 help='specify which special resource intensive tests ' 226 'to run.' + more_details) 227 group.add_argument('-M', '--memlimit', metavar='LIMIT', 228 help='run very large memory-consuming tests.' + 229 more_details) 230 group.add_argument('--testdir', metavar='DIR', 231 type=relative_filename, 232 help='execute test files in the specified directory ' 233 '(instead of the Python stdlib test suite)') 234 235 group = parser.add_argument_group('Special runs') 236 group.add_argument('-l', '--findleaks', action='store_const', const=2, 237 default=1, 238 help='deprecated alias to --fail-env-changed') 239 group.add_argument('-L', '--runleaks', action='store_true', 240 help='run the leaks(1) command just before exit.' + 241 more_details) 242 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', 243 type=huntrleaks, 244 help='search for reference leaks (needs debug build, ' 245 'very slow).' + more_details) 246 group.add_argument('-j', '--multiprocess', metavar='PROCESSES', 247 dest='use_mp', type=int, 248 help='run PROCESSES processes at once') 249 group.add_argument('-T', '--coverage', action='store_true', 250 dest='trace', 251 help='turn on code coverage tracing using the trace ' 252 'module') 253 group.add_argument('-D', '--coverdir', metavar='DIR', 254 type=relative_filename, 255 help='directory where coverage files are put') 256 group.add_argument('-N', '--nocoverdir', 257 action='store_const', const=None, dest='coverdir', 258 help='put coverage files alongside modules') 259 group.add_argument('-t', '--threshold', metavar='THRESHOLD', 260 type=int, 261 help='call gc.set_threshold(THRESHOLD)') 262 group.add_argument('-n', '--nowindows', action='store_true', 263 help='suppress error message boxes on Windows') 264 group.add_argument('-F', '--forever', action='store_true', 265 help='run the specified tests in a loop, until an ' 266 'error happens; imply --failfast') 267 group.add_argument('--list-tests', action='store_true', 268 help="only write the name of tests that will be run, " 269 "don't execute them") 270 group.add_argument('--list-cases', action='store_true', 271 help='only write the name of test cases that will be run' 272 ' , don\'t execute them') 273 group.add_argument('-P', '--pgo', dest='pgo', action='store_true', 274 help='enable Profile Guided Optimization (PGO) training') 275 group.add_argument('--pgo-extended', action='store_true', 276 help='enable extended PGO training (slower training)') 277 group.add_argument('--fail-env-changed', action='store_true', 278 help='if a test file alters the environment, mark ' 279 'the test as failed') 280 281 group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', 282 help='writes JUnit-style XML results to the specified ' 283 'file') 284 group.add_argument('--tempdir', metavar='PATH', 285 help='override the working directory for the test run') 286 group.add_argument('--cleanup', action='store_true', 287 help='remove old test_python_* directories') 288 return parser 289 290 291def relative_filename(string): 292 # CWD is replaced with a temporary dir before calling main(), so we 293 # join it with the saved CWD so it ends up where the user expects. 294 return os.path.join(support.SAVEDCWD, string) 295 296 297def huntrleaks(string): 298 args = string.split(':') 299 if len(args) not in (2, 3): 300 raise argparse.ArgumentTypeError( 301 'needs 2 or 3 colon-separated arguments') 302 nwarmup = int(args[0]) if args[0] else 5 303 ntracked = int(args[1]) if args[1] else 4 304 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' 305 return nwarmup, ntracked, fname 306 307 308def resources_list(string): 309 u = [x.lower() for x in string.split(',')] 310 for r in u: 311 if r == 'all' or r == 'none': 312 continue 313 if r[0] == '-': 314 r = r[1:] 315 if r not in RESOURCE_NAMES: 316 raise argparse.ArgumentTypeError('invalid resource: ' + r) 317 return u 318 319 320def _parse_args(args, **kwargs): 321 # Defaults 322 ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, 323 exclude=False, single=False, randomize=False, fromfile=None, 324 findleaks=1, use_resources=None, trace=False, coverdir='coverage', 325 runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, 326 random_seed=None, use_mp=None, verbose3=False, forever=False, 327 header=False, failfast=False, match_tests=None, ignore_tests=None, 328 pgo=False) 329 for k, v in kwargs.items(): 330 if not hasattr(ns, k): 331 raise TypeError('%r is an invalid keyword argument ' 332 'for this function' % k) 333 setattr(ns, k, v) 334 if ns.use_resources is None: 335 ns.use_resources = [] 336 337 parser = _create_parser() 338 # Issue #14191: argparse doesn't support "intermixed" positional and 339 # optional arguments. Use parse_known_args() as workaround. 340 ns.args = parser.parse_known_args(args=args, namespace=ns)[1] 341 for arg in ns.args: 342 if arg.startswith('-'): 343 parser.error("unrecognized arguments: %s" % arg) 344 sys.exit(1) 345 346 if ns.findleaks > 1: 347 # --findleaks implies --fail-env-changed 348 ns.fail_env_changed = True 349 if ns.single and ns.fromfile: 350 parser.error("-s and -f don't go together!") 351 if ns.use_mp is not None and ns.trace: 352 parser.error("-T and -j don't go together!") 353 if ns.failfast and not (ns.verbose or ns.verbose3): 354 parser.error("-G/--failfast needs either -v or -W") 355 if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): 356 parser.error("--pgo/-v don't go together!") 357 if ns.pgo_extended: 358 ns.pgo = True # pgo_extended implies pgo 359 360 if ns.nowindows: 361 print("Warning: the --nowindows (-n) option is deprecated. " 362 "Use -vv to display assertions in stderr.", file=sys.stderr) 363 364 if ns.quiet: 365 ns.verbose = 0 366 if ns.timeout is not None: 367 if ns.timeout <= 0: 368 ns.timeout = None 369 if ns.use_mp is not None: 370 if ns.use_mp <= 0: 371 # Use all cores + extras for tests that like to sleep 372 ns.use_mp = 2 + (os.cpu_count() or 1) 373 if ns.use: 374 for a in ns.use: 375 for r in a: 376 if r == 'all': 377 ns.use_resources[:] = ALL_RESOURCES 378 continue 379 if r == 'none': 380 del ns.use_resources[:] 381 continue 382 remove = False 383 if r[0] == '-': 384 remove = True 385 r = r[1:] 386 if remove: 387 if r in ns.use_resources: 388 ns.use_resources.remove(r) 389 elif r not in ns.use_resources: 390 ns.use_resources.append(r) 391 if ns.random_seed is not None: 392 ns.randomize = True 393 if ns.verbose: 394 ns.header = True 395 if ns.huntrleaks and ns.verbose3: 396 ns.verbose3 = False 397 print("WARNING: Disable --verbose3 because it's incompatible with " 398 "--huntrleaks: see http://bugs.python.org/issue27103", 399 file=sys.stderr) 400 if ns.match_filename: 401 if ns.match_tests is None: 402 ns.match_tests = [] 403 with open(ns.match_filename) as fp: 404 for line in fp: 405 ns.match_tests.append(line.strip()) 406 if ns.ignore_filename: 407 if ns.ignore_tests is None: 408 ns.ignore_tests = [] 409 with open(ns.ignore_filename) as fp: 410 for line in fp: 411 ns.ignore_tests.append(line.strip()) 412 if ns.forever: 413 # --forever implies --failfast 414 ns.failfast = True 415 416 return ns 417