1"""distutils.util 2 3Miscellaneous utility functions -- anything that doesn't fit into 4one of the other *util.py modules. 5""" 6 7import os 8import re 9import importlib.util 10import string 11import sys 12from distutils.errors import DistutilsPlatformError 13from distutils.dep_util import newer 14from distutils.spawn import spawn 15from distutils import log 16from distutils.errors import DistutilsByteCompileError 17 18def get_platform (): 19 """Return a string that identifies the current platform. This is used 20 mainly to distinguish platform-specific build directories and 21 platform-specific built distributions. Typically includes the OS name 22 and version and the architecture (as supplied by 'os.uname()'), 23 although the exact information included depends on the OS; eg. for IRIX 24 the architecture isn't particularly important (IRIX only runs on SGI 25 hardware), but for Linux the kernel version isn't particularly 26 important. 27 28 Examples of returned values: 29 linux-i586 30 linux-alpha (?) 31 solaris-2.6-sun4u 32 irix-5.3 33 irix64-6.2 34 35 Windows will return one of: 36 win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) 37 win-ia64 (64bit Windows on Itanium) 38 win32 (all others - specifically, sys.platform is returned) 39 40 For other non-POSIX platforms, currently just returns 'sys.platform'. 41 """ 42 if os.name == 'nt': 43 # sniff sys.version for architecture. 44 prefix = " bit (" 45 i = sys.version.find(prefix) 46 if i == -1: 47 return sys.platform 48 j = sys.version.find(")", i) 49 look = sys.version[i+len(prefix):j].lower() 50 if look == 'amd64': 51 return 'win-amd64' 52 if look == 'itanium': 53 return 'win-ia64' 54 return sys.platform 55 56 # Set for cross builds explicitly 57 if "_PYTHON_HOST_PLATFORM" in os.environ: 58 return os.environ["_PYTHON_HOST_PLATFORM"] 59 60 if os.name != "posix" or not hasattr(os, 'uname'): 61 # XXX what about the architecture? NT is Intel or Alpha, 62 # Mac OS is M68k or PPC, etc. 63 return sys.platform 64 65 # Try to distinguish various flavours of Unix 66 67 (osname, host, release, version, machine) = os.uname() 68 69 # Convert the OS name to lowercase, remove '/' characters 70 # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh") 71 osname = osname.lower().replace('/', '') 72 machine = machine.replace(' ', '_') 73 machine = machine.replace('/', '-') 74 75 if osname[:5] == "linux": 76 # At least on Linux/Intel, 'machine' is the processor -- 77 # i386, etc. 78 # XXX what about Alpha, SPARC, etc? 79 return "%s-%s" % (osname, machine) 80 elif osname[:5] == "sunos": 81 if release[0] >= "5": # SunOS 5 == Solaris 2 82 osname = "solaris" 83 release = "%d.%s" % (int(release[0]) - 3, release[2:]) 84 # We can't use "platform.architecture()[0]" because a 85 # bootstrap problem. We use a dict to get an error 86 # if some suspicious happens. 87 bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} 88 machine += ".%s" % bitness[sys.maxsize] 89 # fall through to standard osname-release-machine representation 90 elif osname[:4] == "irix": # could be "irix64"! 91 return "%s-%s" % (osname, release) 92 elif osname[:3] == "aix": 93 return "%s-%s.%s" % (osname, version, release) 94 elif osname[:6] == "cygwin": 95 osname = "cygwin" 96 rel_re = re.compile (r'[\d.]+', re.ASCII) 97 m = rel_re.match(release) 98 if m: 99 release = m.group() 100 elif osname[:6] == "darwin": 101 import _osx_support, distutils.sysconfig 102 osname, release, machine = _osx_support.get_platform_osx( 103 distutils.sysconfig.get_config_vars(), 104 osname, release, machine) 105 106 return "%s-%s-%s" % (osname, release, machine) 107 108# get_platform () 109 110 111def convert_path (pathname): 112 """Return 'pathname' as a name that will work on the native filesystem, 113 i.e. split it on '/' and put it back together again using the current 114 directory separator. Needed because filenames in the setup script are 115 always supplied in Unix style, and have to be converted to the local 116 convention before we can actually use them in the filesystem. Raises 117 ValueError on non-Unix-ish systems if 'pathname' either starts or 118 ends with a slash. 119 """ 120 if os.sep == '/': 121 return pathname 122 if not pathname: 123 return pathname 124 if pathname[0] == '/': 125 raise ValueError("path '%s' cannot be absolute" % pathname) 126 if pathname[-1] == '/': 127 raise ValueError("path '%s' cannot end with '/'" % pathname) 128 129 paths = pathname.split('/') 130 while '.' in paths: 131 paths.remove('.') 132 if not paths: 133 return os.curdir 134 return os.path.join(*paths) 135 136# convert_path () 137 138 139def change_root (new_root, pathname): 140 """Return 'pathname' with 'new_root' prepended. If 'pathname' is 141 relative, this is equivalent to "os.path.join(new_root,pathname)". 142 Otherwise, it requires making 'pathname' relative and then joining the 143 two, which is tricky on DOS/Windows and Mac OS. 144 """ 145 if os.name == 'posix': 146 if not os.path.isabs(pathname): 147 return os.path.join(new_root, pathname) 148 else: 149 return os.path.join(new_root, pathname[1:]) 150 151 elif os.name == 'nt': 152 (drive, path) = os.path.splitdrive(pathname) 153 if path[0] == '\\': 154 path = path[1:] 155 return os.path.join(new_root, path) 156 157 else: 158 raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) 159 160 161_environ_checked = 0 162def check_environ (): 163 """Ensure that 'os.environ' has all the environment variables we 164 guarantee that users can use in config files, command-line options, 165 etc. Currently this includes: 166 HOME - user's home directory (Unix only) 167 PLAT - description of the current platform, including hardware 168 and OS (see 'get_platform()') 169 """ 170 global _environ_checked 171 if _environ_checked: 172 return 173 174 if os.name == 'posix' and 'HOME' not in os.environ: 175 import pwd 176 os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] 177 178 if 'PLAT' not in os.environ: 179 os.environ['PLAT'] = get_platform() 180 181 _environ_checked = 1 182 183 184def subst_vars (s, local_vars): 185 """Perform shell/Perl-style variable substitution on 'string'. Every 186 occurrence of '$' followed by a name is considered a variable, and 187 variable is substituted by the value found in the 'local_vars' 188 dictionary, or in 'os.environ' if it's not in 'local_vars'. 189 'os.environ' is first checked/augmented to guarantee that it contains 190 certain values: see 'check_environ()'. Raise ValueError for any 191 variables not found in either 'local_vars' or 'os.environ'. 192 """ 193 check_environ() 194 def _subst (match, local_vars=local_vars): 195 var_name = match.group(1) 196 if var_name in local_vars: 197 return str(local_vars[var_name]) 198 else: 199 return os.environ[var_name] 200 201 try: 202 return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) 203 except KeyError as var: 204 raise ValueError("invalid variable '$%s'" % var) 205 206# subst_vars () 207 208 209def grok_environment_error (exc, prefix="error: "): 210 # Function kept for backward compatibility. 211 # Used to try clever things with EnvironmentErrors, 212 # but nowadays str(exception) produces good messages. 213 return prefix + str(exc) 214 215 216# Needed by 'split_quoted()' 217_wordchars_re = _squote_re = _dquote_re = None 218def _init_regex(): 219 global _wordchars_re, _squote_re, _dquote_re 220 _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) 221 _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") 222 _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') 223 224def split_quoted (s): 225 """Split a string up according to Unix shell-like rules for quotes and 226 backslashes. In short: words are delimited by spaces, as long as those 227 spaces are not escaped by a backslash, or inside a quoted string. 228 Single and double quotes are equivalent, and the quote characters can 229 be backslash-escaped. The backslash is stripped from any two-character 230 escape sequence, leaving only the escaped character. The quote 231 characters are stripped from any quoted string. Returns a list of 232 words. 233 """ 234 235 # This is a nice algorithm for splitting up a single string, since it 236 # doesn't require character-by-character examination. It was a little 237 # bit of a brain-bender to get it working right, though... 238 if _wordchars_re is None: _init_regex() 239 240 s = s.strip() 241 words = [] 242 pos = 0 243 244 while s: 245 m = _wordchars_re.match(s, pos) 246 end = m.end() 247 if end == len(s): 248 words.append(s[:end]) 249 break 250 251 if s[end] in string.whitespace: # unescaped, unquoted whitespace: now 252 words.append(s[:end]) # we definitely have a word delimiter 253 s = s[end:].lstrip() 254 pos = 0 255 256 elif s[end] == '\\': # preserve whatever is being escaped; 257 # will become part of the current word 258 s = s[:end] + s[end+1:] 259 pos = end+1 260 261 else: 262 if s[end] == "'": # slurp singly-quoted string 263 m = _squote_re.match(s, end) 264 elif s[end] == '"': # slurp doubly-quoted string 265 m = _dquote_re.match(s, end) 266 else: 267 raise RuntimeError("this can't happen (bad char '%c')" % s[end]) 268 269 if m is None: 270 raise ValueError("bad string (mismatched %s quotes?)" % s[end]) 271 272 (beg, end) = m.span() 273 s = s[:beg] + s[beg+1:end-1] + s[end:] 274 pos = m.end() - 2 275 276 if pos >= len(s): 277 words.append(s) 278 break 279 280 return words 281 282# split_quoted () 283 284 285def execute (func, args, msg=None, verbose=0, dry_run=0): 286 """Perform some action that affects the outside world (eg. by 287 writing to the filesystem). Such actions are special because they 288 are disabled by the 'dry_run' flag. This method takes care of all 289 that bureaucracy for you; all you have to do is supply the 290 function to call and an argument tuple for it (to embody the 291 "external action" being performed), and an optional message to 292 print. 293 """ 294 if msg is None: 295 msg = "%s%r" % (func.__name__, args) 296 if msg[-2:] == ',)': # correct for singleton tuple 297 msg = msg[0:-2] + ')' 298 299 log.info(msg) 300 if not dry_run: 301 func(*args) 302 303 304def strtobool (val): 305 """Convert a string representation of truth to true (1) or false (0). 306 307 True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values 308 are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if 309 'val' is anything else. 310 """ 311 val = val.lower() 312 if val in ('y', 'yes', 't', 'true', 'on', '1'): 313 return 1 314 elif val in ('n', 'no', 'f', 'false', 'off', '0'): 315 return 0 316 else: 317 raise ValueError("invalid truth value %r" % (val,)) 318 319 320def byte_compile (py_files, 321 optimize=0, force=0, 322 prefix=None, base_dir=None, 323 verbose=1, dry_run=0, 324 direct=None): 325 """Byte-compile a collection of Python source files to .pyc 326 files in a __pycache__ subdirectory. 'py_files' is a list 327 of files to compile; any files that don't end in ".py" are silently 328 skipped. 'optimize' must be one of the following: 329 0 - don't optimize 330 1 - normal optimization (like "python -O") 331 2 - extra optimization (like "python -OO") 332 If 'force' is true, all files are recompiled regardless of 333 timestamps. 334 335 The source filename encoded in each bytecode file defaults to the 336 filenames listed in 'py_files'; you can modify these with 'prefix' and 337 'basedir'. 'prefix' is a string that will be stripped off of each 338 source filename, and 'base_dir' is a directory name that will be 339 prepended (after 'prefix' is stripped). You can supply either or both 340 (or neither) of 'prefix' and 'base_dir', as you wish. 341 342 If 'dry_run' is true, doesn't actually do anything that would 343 affect the filesystem. 344 345 Byte-compilation is either done directly in this interpreter process 346 with the standard py_compile module, or indirectly by writing a 347 temporary script and executing it. Normally, you should let 348 'byte_compile()' figure out to use direct compilation or not (see 349 the source for details). The 'direct' flag is used by the script 350 generated in indirect mode; unless you know what you're doing, leave 351 it set to None. 352 """ 353 354 # Late import to fix a bootstrap issue: _posixsubprocess is built by 355 # setup.py, but setup.py uses distutils. 356 import subprocess 357 358 # nothing is done if sys.dont_write_bytecode is True 359 if sys.dont_write_bytecode: 360 raise DistutilsByteCompileError('byte-compiling is disabled.') 361 362 # First, if the caller didn't force us into direct or indirect mode, 363 # figure out which mode we should be in. We take a conservative 364 # approach: choose direct mode *only* if the current interpreter is 365 # in debug mode and optimize is 0. If we're not in debug mode (-O 366 # or -OO), we don't know which level of optimization this 367 # interpreter is running with, so we can't do direct 368 # byte-compilation and be certain that it's the right thing. Thus, 369 # always compile indirectly if the current interpreter is in either 370 # optimize mode, or if either optimization level was requested by 371 # the caller. 372 if direct is None: 373 direct = (__debug__ and optimize == 0) 374 375 # "Indirect" byte-compilation: write a temporary script and then 376 # run it with the appropriate flags. 377 if not direct: 378 try: 379 from tempfile import mkstemp 380 (script_fd, script_name) = mkstemp(".py") 381 except ImportError: 382 from tempfile import mktemp 383 (script_fd, script_name) = None, mktemp(".py") 384 log.info("writing byte-compilation script '%s'", script_name) 385 if not dry_run: 386 if script_fd is not None: 387 script = os.fdopen(script_fd, "w") 388 else: 389 script = open(script_name, "w") 390 391 script.write("""\ 392from distutils.util import byte_compile 393files = [ 394""") 395 396 # XXX would be nice to write absolute filenames, just for 397 # safety's sake (script should be more robust in the face of 398 # chdir'ing before running it). But this requires abspath'ing 399 # 'prefix' as well, and that breaks the hack in build_lib's 400 # 'byte_compile()' method that carefully tacks on a trailing 401 # slash (os.sep really) to make sure the prefix here is "just 402 # right". This whole prefix business is rather delicate -- the 403 # problem is that it's really a directory, but I'm treating it 404 # as a dumb string, so trailing slashes and so forth matter. 405 406 #py_files = map(os.path.abspath, py_files) 407 #if prefix: 408 # prefix = os.path.abspath(prefix) 409 410 script.write(",\n".join(map(repr, py_files)) + "]\n") 411 script.write(""" 412byte_compile(files, optimize=%r, force=%r, 413 prefix=%r, base_dir=%r, 414 verbose=%r, dry_run=0, 415 direct=1) 416""" % (optimize, force, prefix, base_dir, verbose)) 417 418 script.close() 419 420 cmd = [sys.executable] 421 cmd.extend(subprocess._optim_args_from_interpreter_flags()) 422 cmd.append(script_name) 423 spawn(cmd, dry_run=dry_run) 424 execute(os.remove, (script_name,), "removing %s" % script_name, 425 dry_run=dry_run) 426 427 # "Direct" byte-compilation: use the py_compile module to compile 428 # right here, right now. Note that the script generated in indirect 429 # mode simply calls 'byte_compile()' in direct mode, a weird sort of 430 # cross-process recursion. Hey, it works! 431 else: 432 from py_compile import compile 433 434 for file in py_files: 435 if file[-3:] != ".py": 436 # This lets us be lazy and not filter filenames in 437 # the "install_lib" command. 438 continue 439 440 # Terminology from the py_compile module: 441 # cfile - byte-compiled file 442 # dfile - purported source filename (same as 'file' by default) 443 if optimize >= 0: 444 opt = '' if optimize == 0 else optimize 445 cfile = importlib.util.cache_from_source( 446 file, optimization=opt) 447 else: 448 cfile = importlib.util.cache_from_source(file) 449 dfile = file 450 if prefix: 451 if file[:len(prefix)] != prefix: 452 raise ValueError("invalid prefix: filename %r doesn't start with %r" 453 % (file, prefix)) 454 dfile = dfile[len(prefix):] 455 if base_dir: 456 dfile = os.path.join(base_dir, dfile) 457 458 cfile_base = os.path.basename(cfile) 459 if direct: 460 if force or newer(file, cfile): 461 log.info("byte-compiling %s to %s", file, cfile_base) 462 if not dry_run: 463 compile(file, cfile, dfile) 464 else: 465 log.debug("skipping byte-compilation of %s to %s", 466 file, cfile_base) 467 468# byte_compile () 469 470def rfc822_escape (header): 471 """Return a version of the string escaped for inclusion in an 472 RFC-822 header, by ensuring there are 8 spaces space after each newline. 473 """ 474 lines = header.split('\n') 475 sep = '\n' + 8 * ' ' 476 return sep.join(lines) 477 478# 2to3 support 479 480def run_2to3(files, fixer_names=None, options=None, explicit=None): 481 """Invoke 2to3 on a list of Python files. 482 The files should all come from the build area, as the 483 modification is done in-place. To reduce the build time, 484 only files modified since the last invocation of this 485 function should be passed in the files argument.""" 486 487 if not files: 488 return 489 490 # Make this class local, to delay import of 2to3 491 from lib2to3.refactor import RefactoringTool, get_fixers_from_package 492 class DistutilsRefactoringTool(RefactoringTool): 493 def log_error(self, msg, *args, **kw): 494 log.error(msg, *args) 495 496 def log_message(self, msg, *args): 497 log.info(msg, *args) 498 499 def log_debug(self, msg, *args): 500 log.debug(msg, *args) 501 502 if fixer_names is None: 503 fixer_names = get_fixers_from_package('lib2to3.fixes') 504 r = DistutilsRefactoringTool(fixer_names, options=options) 505 r.refactor(files, write=True) 506 507def copydir_run_2to3(src, dest, template=None, fixer_names=None, 508 options=None, explicit=None): 509 """Recursively copy a directory, only copying new and changed files, 510 running run_2to3 over all newly copied Python modules afterward. 511 512 If you give a template string, it's parsed like a MANIFEST.in. 513 """ 514 from distutils.dir_util import mkpath 515 from distutils.file_util import copy_file 516 from distutils.filelist import FileList 517 filelist = FileList() 518 curdir = os.getcwd() 519 os.chdir(src) 520 try: 521 filelist.findall() 522 finally: 523 os.chdir(curdir) 524 filelist.files[:] = filelist.allfiles 525 if template: 526 for line in template.splitlines(): 527 line = line.strip() 528 if not line: continue 529 filelist.process_template_line(line) 530 copied = [] 531 for filename in filelist.files: 532 outname = os.path.join(dest, filename) 533 mkpath(os.path.dirname(outname)) 534 res = copy_file(os.path.join(src, filename), outname, update=1) 535 if res[1]: copied.append(outname) 536 run_2to3([fn for fn in copied if fn.lower().endswith('.py')], 537 fixer_names=fixer_names, options=options, explicit=explicit) 538 return copied 539 540class Mixin2to3: 541 '''Mixin class for commands that run 2to3. 542 To configure 2to3, setup scripts may either change 543 the class variables, or inherit from individual commands 544 to override how 2to3 is invoked.''' 545 546 # provide list of fixers to run; 547 # defaults to all from lib2to3.fixers 548 fixer_names = None 549 550 # options dictionary 551 options = None 552 553 # list of fixers to invoke even though they are marked as explicit 554 explicit = None 555 556 def run_2to3(self, files): 557 return run_2to3(files, self.fixer_names, self.options, self.explicit) 558