1import argparse 2import os 3import sys 4from test import support 5from test.support import os_helper 6 7 8USAGE = """\ 9python -m test [options] [test_name1 [test_name2 ...]] 10python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] 11""" 12 13DESCRIPTION = """\ 14Run Python regression tests. 15 16If no arguments or options are provided, finds all files matching 17the pattern "test_*" in the Lib/test subdirectory and runs 18them in alphabetical order (but see -M and -u, below, for exceptions). 19 20For more rigorous testing, it is useful to use the following 21command line: 22 23python -E -Wd -m test [options] [test_name1 ...] 24""" 25 26EPILOG = """\ 27Additional option details: 28 29-r randomizes test execution order. You can use --randseed=int to provide an 30int seed value for the randomizer; this is useful for reproducing troublesome 31test orders. 32 33-s On the first invocation of regrtest using -s, the first test file found 34or the first test file given on the command line is run, and the name of 35the next test is recorded in a file named pynexttest. If run from the 36Python build directory, pynexttest is located in the 'build' subdirectory, 37otherwise it is located in tempfile.gettempdir(). On subsequent runs, 38the test in pynexttest is run, and the next test is written to pynexttest. 39When the last test has been run, pynexttest is deleted. In this way it 40is possible to single step through the test files. This is useful when 41doing memory analysis on the Python interpreter, which process tends to 42consume too many resources to run the full regression test non-stop. 43 44-S is used to continue running tests after an aborted run. It will 45maintain the order a standard run (ie, this assumes -r is not used). 46This is useful after the tests have prematurely stopped for some external 47reason and you want to start running from where you left off rather 48than starting from the beginning. 49 50-f reads the names of tests from the file given as f's argument, one 51or more test names per line. Whitespace is ignored. Blank lines and 52lines beginning with '#' are ignored. This is especially useful for 53whittling down failures involving interactions among tests. 54 55-L causes the leaks(1) command to be run just before exit if it exists. 56leaks(1) is available on Mac OS X and presumably on some other 57FreeBSD-derived systems. 58 59-R runs each test several times and examines sys.gettotalrefcount() to 60see if the test appears to be leaking references. The argument should 61be of the form stab:run:fname where 'stab' is the number of times the 62test is run to let gettotalrefcount settle down, 'run' is the number 63of times further it is run and 'fname' is the name of the file the 64reports are written to. These parameters all have defaults (5, 4 and 65"reflog.txt" respectively), and the minimal invocation is '-R :'. 66 67-M runs tests that require an exorbitant amount of memory. These tests 68typically try to ascertain containers keep working when containing more than 692 billion objects, which only works on 64-bit systems. There are also some 70tests that try to exhaust the address space of the process, which only makes 71sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, 72which is a string in the form of '2.5Gb', determines how much memory the 73tests will limit themselves to (but they may go slightly over.) The number 74shouldn't be more memory than the machine has (including swap memory). You 75should also keep in mind that swap memory is generally much, much slower 76than RAM, and setting memlimit to all available RAM or higher will heavily 77tax the machine. On the other hand, it is no use running these tests with a 78limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect 79to use more than memlimit memory will be skipped. The big-memory tests 80generally run very, very long. 81 82-u is used to specify which special resource intensive tests to run, 83such as those requiring large file support or network connectivity. 84The argument is a comma-separated list of words indicating the 85resources to test. Currently only the following are defined: 86 87 all - Enable all special resources. 88 89 none - Disable all special resources (this is the default). 90 91 audio - Tests that use the audio device. (There are known 92 cases of broken audio drivers that can crash Python or 93 even the Linux kernel.) 94 95 curses - Tests that use curses and will modify the terminal's 96 state and output modes. 97 98 largefile - It is okay to run some test that may create huge 99 files. These tests can take a long time and may 100 consume >2 GiB of disk space temporarily. 101 102 network - It is okay to run tests that use external network 103 resource, e.g. testing SSL support for sockets. 104 105 decimal - Test the decimal module against a large suite that 106 verifies compliance with standards. 107 108 cpu - Used for certain CPU-heavy tests. 109 110 subprocess Run all tests for the subprocess module. 111 112 urlfetch - It is okay to download files required on testing. 113 114 gui - Run tests that require a running GUI. 115 116 tzdata - Run tests that require timezone data. 117 118To enable all resources except one, use '-uall,-<resource>'. For 119example, to run all the tests except for the gui tests, give the 120option '-uall,-gui'. 121 122--matchfile filters tests using a text file, one pattern per line. 123Pattern examples: 124 125- test method: test_stat_attributes 126- test class: FileTests 127- test identifier: test_os.FileTests.test_stat_attributes 128""" 129 130 131ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', 132 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') 133 134# Other resources excluded from --use=all: 135# 136# - extralagefile (ex: test_zipfile64): really too slow to be enabled 137# "by default" 138# - tzdata: while needed to validate fully test_datetime, it makes 139# test_datetime too slow (15-20 min on some buildbots) and so is disabled by 140# default (see bpo-30822). 141RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') 142 143 144class Namespace(argparse.Namespace): 145 def __init__(self, **kwargs) -> None: 146 self.testdir = None 147 self.verbose = 0 148 self.quiet = False 149 self.exclude = False 150 self.single = False 151 self.randomize = False 152 self.fromfile = None 153 self.findleaks = 1 154 self.fail_env_changed = False 155 self.use_resources = None 156 self.trace = False 157 self.coverdir = 'coverage' 158 self.runleaks = False 159 self.huntrleaks = False 160 self.verbose2 = False 161 self.verbose3 = False 162 self.print_slow = False 163 self.random_seed = None 164 self.use_mp = None 165 self.forever = False 166 self.header = False 167 self.failfast = False 168 self.match_tests = None 169 self.ignore_tests = None 170 self.pgo = False 171 self.pgo_extended = False 172 173 super().__init__(**kwargs) 174 175 176class _ArgParser(argparse.ArgumentParser): 177 178 def error(self, message): 179 super().error(message + "\nPass -h or --help for complete help.") 180 181 182def _create_parser(): 183 # Set prog to prevent the uninformative "__main__.py" from displaying in 184 # error messages when using "python -m test ...". 185 parser = _ArgParser(prog='regrtest.py', 186 usage=USAGE, 187 description=DESCRIPTION, 188 epilog=EPILOG, 189 add_help=False, 190 formatter_class=argparse.RawDescriptionHelpFormatter) 191 192 # Arguments with this clause added to its help are described further in 193 # the epilog's "Additional option details" section. 194 more_details = ' See the section at bottom for more details.' 195 196 group = parser.add_argument_group('General options') 197 # We add help explicitly to control what argument group it renders under. 198 group.add_argument('-h', '--help', action='help', 199 help='show this help message and exit') 200 group.add_argument('--timeout', metavar='TIMEOUT', type=float, 201 help='dump the traceback and exit if a test takes ' 202 'more than TIMEOUT seconds; disabled if TIMEOUT ' 203 'is negative or equals to zero') 204 group.add_argument('--wait', action='store_true', 205 help='wait for user input, e.g., allow a debugger ' 206 'to be attached') 207 group.add_argument('--worker-args', metavar='ARGS') 208 group.add_argument('-S', '--start', metavar='START', 209 help='the name of the test at which to start.' + 210 more_details) 211 212 group = parser.add_argument_group('Verbosity') 213 group.add_argument('-v', '--verbose', action='count', 214 help='run tests in verbose mode with output to stdout') 215 group.add_argument('-w', '--verbose2', action='store_true', 216 help='re-run failed tests in verbose mode') 217 group.add_argument('-W', '--verbose3', action='store_true', 218 help='display test output on failure') 219 group.add_argument('-q', '--quiet', action='store_true', 220 help='no output unless one or more tests fail') 221 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow', 222 help='print the slowest 10 tests') 223 group.add_argument('--header', action='store_true', 224 help='print header with interpreter info') 225 226 group = parser.add_argument_group('Selecting tests') 227 group.add_argument('-r', '--randomize', action='store_true', 228 help='randomize test execution order.' + more_details) 229 group.add_argument('--randseed', metavar='SEED', 230 dest='random_seed', type=int, 231 help='pass a random seed to reproduce a previous ' 232 'random run') 233 group.add_argument('-f', '--fromfile', metavar='FILE', 234 help='read names of tests to run from a file.' + 235 more_details) 236 group.add_argument('-x', '--exclude', action='store_true', 237 help='arguments are tests to *exclude*') 238 group.add_argument('-s', '--single', action='store_true', 239 help='single step through a set of tests.' + 240 more_details) 241 group.add_argument('-m', '--match', metavar='PAT', 242 dest='match_tests', action='append', 243 help='match test cases and methods with glob pattern PAT') 244 group.add_argument('-i', '--ignore', metavar='PAT', 245 dest='ignore_tests', action='append', 246 help='ignore test cases and methods with glob pattern PAT') 247 group.add_argument('--matchfile', metavar='FILENAME', 248 dest='match_filename', 249 help='similar to --match but get patterns from a ' 250 'text file, one pattern per line') 251 group.add_argument('--ignorefile', metavar='FILENAME', 252 dest='ignore_filename', 253 help='similar to --matchfile but it receives patterns ' 254 'from text file to ignore') 255 group.add_argument('-G', '--failfast', action='store_true', 256 help='fail as soon as a test fails (only with -v or -W)') 257 group.add_argument('-u', '--use', metavar='RES1,RES2,...', 258 action='append', type=resources_list, 259 help='specify which special resource intensive tests ' 260 'to run.' + more_details) 261 group.add_argument('-M', '--memlimit', metavar='LIMIT', 262 help='run very large memory-consuming tests.' + 263 more_details) 264 group.add_argument('--testdir', metavar='DIR', 265 type=relative_filename, 266 help='execute test files in the specified directory ' 267 '(instead of the Python stdlib test suite)') 268 269 group = parser.add_argument_group('Special runs') 270 group.add_argument('-l', '--findleaks', action='store_const', const=2, 271 default=1, 272 help='deprecated alias to --fail-env-changed') 273 group.add_argument('-L', '--runleaks', action='store_true', 274 help='run the leaks(1) command just before exit.' + 275 more_details) 276 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', 277 type=huntrleaks, 278 help='search for reference leaks (needs debug build, ' 279 'very slow).' + more_details) 280 group.add_argument('-j', '--multiprocess', metavar='PROCESSES', 281 dest='use_mp', type=int, 282 help='run PROCESSES processes at once') 283 group.add_argument('-T', '--coverage', action='store_true', 284 dest='trace', 285 help='turn on code coverage tracing using the trace ' 286 'module') 287 group.add_argument('-D', '--coverdir', metavar='DIR', 288 type=relative_filename, 289 help='directory where coverage files are put') 290 group.add_argument('-N', '--nocoverdir', 291 action='store_const', const=None, dest='coverdir', 292 help='put coverage files alongside modules') 293 group.add_argument('-t', '--threshold', metavar='THRESHOLD', 294 type=int, 295 help='call gc.set_threshold(THRESHOLD)') 296 group.add_argument('-n', '--nowindows', action='store_true', 297 help='suppress error message boxes on Windows') 298 group.add_argument('-F', '--forever', action='store_true', 299 help='run the specified tests in a loop, until an ' 300 'error happens; imply --failfast') 301 group.add_argument('--list-tests', action='store_true', 302 help="only write the name of tests that will be run, " 303 "don't execute them") 304 group.add_argument('--list-cases', action='store_true', 305 help='only write the name of test cases that will be run' 306 ' , don\'t execute them') 307 group.add_argument('-P', '--pgo', dest='pgo', action='store_true', 308 help='enable Profile Guided Optimization (PGO) training') 309 group.add_argument('--pgo-extended', action='store_true', 310 help='enable extended PGO training (slower training)') 311 group.add_argument('--fail-env-changed', action='store_true', 312 help='if a test file alters the environment, mark ' 313 'the test as failed') 314 315 group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', 316 help='writes JUnit-style XML results to the specified ' 317 'file') 318 group.add_argument('--tempdir', metavar='PATH', 319 help='override the working directory for the test run') 320 group.add_argument('--cleanup', action='store_true', 321 help='remove old test_python_* directories') 322 return parser 323 324 325def relative_filename(string): 326 # CWD is replaced with a temporary dir before calling main(), so we 327 # join it with the saved CWD so it ends up where the user expects. 328 return os.path.join(os_helper.SAVEDCWD, string) 329 330 331def huntrleaks(string): 332 args = string.split(':') 333 if len(args) not in (2, 3): 334 raise argparse.ArgumentTypeError( 335 'needs 2 or 3 colon-separated arguments') 336 nwarmup = int(args[0]) if args[0] else 5 337 ntracked = int(args[1]) if args[1] else 4 338 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' 339 return nwarmup, ntracked, fname 340 341 342def resources_list(string): 343 u = [x.lower() for x in string.split(',')] 344 for r in u: 345 if r == 'all' or r == 'none': 346 continue 347 if r[0] == '-': 348 r = r[1:] 349 if r not in RESOURCE_NAMES: 350 raise argparse.ArgumentTypeError('invalid resource: ' + r) 351 return u 352 353 354def _parse_args(args, **kwargs): 355 # Defaults 356 ns = Namespace() 357 for k, v in kwargs.items(): 358 if not hasattr(ns, k): 359 raise TypeError('%r is an invalid keyword argument ' 360 'for this function' % k) 361 setattr(ns, k, v) 362 if ns.use_resources is None: 363 ns.use_resources = [] 364 365 parser = _create_parser() 366 # Issue #14191: argparse doesn't support "intermixed" positional and 367 # optional arguments. Use parse_known_args() as workaround. 368 ns.args = parser.parse_known_args(args=args, namespace=ns)[1] 369 for arg in ns.args: 370 if arg.startswith('-'): 371 parser.error("unrecognized arguments: %s" % arg) 372 sys.exit(1) 373 374 if ns.findleaks > 1: 375 # --findleaks implies --fail-env-changed 376 ns.fail_env_changed = True 377 if ns.single and ns.fromfile: 378 parser.error("-s and -f don't go together!") 379 if ns.use_mp is not None and ns.trace: 380 parser.error("-T and -j don't go together!") 381 if ns.failfast and not (ns.verbose or ns.verbose3): 382 parser.error("-G/--failfast needs either -v or -W") 383 if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): 384 parser.error("--pgo/-v don't go together!") 385 if ns.pgo_extended: 386 ns.pgo = True # pgo_extended implies pgo 387 388 if ns.nowindows: 389 print("Warning: the --nowindows (-n) option is deprecated. " 390 "Use -vv to display assertions in stderr.", file=sys.stderr) 391 392 if ns.quiet: 393 ns.verbose = 0 394 if ns.timeout is not None: 395 if ns.timeout <= 0: 396 ns.timeout = None 397 if ns.use_mp is not None: 398 if ns.use_mp <= 0: 399 # Use all cores + extras for tests that like to sleep 400 ns.use_mp = 2 + (os.cpu_count() or 1) 401 if ns.use: 402 for a in ns.use: 403 for r in a: 404 if r == 'all': 405 ns.use_resources[:] = ALL_RESOURCES 406 continue 407 if r == 'none': 408 del ns.use_resources[:] 409 continue 410 remove = False 411 if r[0] == '-': 412 remove = True 413 r = r[1:] 414 if remove: 415 if r in ns.use_resources: 416 ns.use_resources.remove(r) 417 elif r not in ns.use_resources: 418 ns.use_resources.append(r) 419 if ns.random_seed is not None: 420 ns.randomize = True 421 if ns.verbose: 422 ns.header = True 423 if ns.huntrleaks and ns.verbose3: 424 ns.verbose3 = False 425 print("WARNING: Disable --verbose3 because it's incompatible with " 426 "--huntrleaks: see http://bugs.python.org/issue27103", 427 file=sys.stderr) 428 if ns.match_filename: 429 if ns.match_tests is None: 430 ns.match_tests = [] 431 with open(ns.match_filename) as fp: 432 for line in fp: 433 ns.match_tests.append(line.strip()) 434 if ns.ignore_filename: 435 if ns.ignore_tests is None: 436 ns.ignore_tests = [] 437 with open(ns.ignore_filename) as fp: 438 for line in fp: 439 ns.ignore_tests.append(line.strip()) 440 if ns.forever: 441 # --forever implies --failfast 442 ns.failfast = True 443 444 return ns 445