1"""setuptools.command.bdist_egg 2 3Build .egg distributions""" 4 5from distutils.errors import DistutilsSetupError 6from distutils.dir_util import remove_tree, mkpath 7from distutils import log 8from types import CodeType 9import sys 10import os 11import re 12import textwrap 13import marshal 14 15from setuptools.extern import six 16 17from pkg_resources import get_build_platform, Distribution, ensure_directory 18from pkg_resources import EntryPoint 19from setuptools.extension import Library 20from setuptools import Command 21 22try: 23 # Python 2.7 or >=3.2 24 from sysconfig import get_path, get_python_version 25 26 def _get_purelib(): 27 return get_path("purelib") 28except ImportError: 29 from distutils.sysconfig import get_python_lib, get_python_version 30 31 def _get_purelib(): 32 return get_python_lib(False) 33 34 35def strip_module(filename): 36 if '.' in filename: 37 filename = os.path.splitext(filename)[0] 38 if filename.endswith('module'): 39 filename = filename[:-6] 40 return filename 41 42 43def sorted_walk(dir): 44 """Do os.walk in a reproducible way, 45 independent of indeterministic filesystem readdir order 46 """ 47 for base, dirs, files in os.walk(dir): 48 dirs.sort() 49 files.sort() 50 yield base, dirs, files 51 52 53def write_stub(resource, pyfile): 54 _stub_template = textwrap.dedent(""" 55 def __bootstrap__(): 56 global __bootstrap__, __loader__, __file__ 57 import sys, pkg_resources, imp 58 __file__ = pkg_resources.resource_filename(__name__, %r) 59 __loader__ = None; del __bootstrap__, __loader__ 60 imp.load_dynamic(__name__,__file__) 61 __bootstrap__() 62 """).lstrip() 63 with open(pyfile, 'w') as f: 64 f.write(_stub_template % resource) 65 66 67class bdist_egg(Command): 68 description = "create an \"egg\" distribution" 69 70 user_options = [ 71 ('bdist-dir=', 'b', 72 "temporary directory for creating the distribution"), 73 ('plat-name=', 'p', "platform name to embed in generated filenames " 74 "(default: %s)" % get_build_platform()), 75 ('exclude-source-files', None, 76 "remove all .py files from the generated egg"), 77 ('keep-temp', 'k', 78 "keep the pseudo-installation tree around after " + 79 "creating the distribution archive"), 80 ('dist-dir=', 'd', 81 "directory to put final built distributions in"), 82 ('skip-build', None, 83 "skip rebuilding everything (for testing/debugging)"), 84 ] 85 86 boolean_options = [ 87 'keep-temp', 'skip-build', 'exclude-source-files' 88 ] 89 90 def initialize_options(self): 91 self.bdist_dir = None 92 self.plat_name = None 93 self.keep_temp = 0 94 self.dist_dir = None 95 self.skip_build = 0 96 self.egg_output = None 97 self.exclude_source_files = None 98 99 def finalize_options(self): 100 ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info") 101 self.egg_info = ei_cmd.egg_info 102 103 if self.bdist_dir is None: 104 bdist_base = self.get_finalized_command('bdist').bdist_base 105 self.bdist_dir = os.path.join(bdist_base, 'egg') 106 107 if self.plat_name is None: 108 self.plat_name = get_build_platform() 109 110 self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) 111 112 if self.egg_output is None: 113 114 # Compute filename of the output egg 115 basename = Distribution( 116 None, None, ei_cmd.egg_name, ei_cmd.egg_version, 117 get_python_version(), 118 self.distribution.has_ext_modules() and self.plat_name 119 ).egg_name() 120 121 self.egg_output = os.path.join(self.dist_dir, basename + '.egg') 122 123 def do_install_data(self): 124 # Hack for packages that install data to install's --install-lib 125 self.get_finalized_command('install').install_lib = self.bdist_dir 126 127 site_packages = os.path.normcase(os.path.realpath(_get_purelib())) 128 old, self.distribution.data_files = self.distribution.data_files, [] 129 130 for item in old: 131 if isinstance(item, tuple) and len(item) == 2: 132 if os.path.isabs(item[0]): 133 realpath = os.path.realpath(item[0]) 134 normalized = os.path.normcase(realpath) 135 if normalized == site_packages or normalized.startswith( 136 site_packages + os.sep 137 ): 138 item = realpath[len(site_packages) + 1:], item[1] 139 # XXX else: raise ??? 140 self.distribution.data_files.append(item) 141 142 try: 143 log.info("installing package data to %s", self.bdist_dir) 144 self.call_command('install_data', force=0, root=None) 145 finally: 146 self.distribution.data_files = old 147 148 def get_outputs(self): 149 return [self.egg_output] 150 151 def call_command(self, cmdname, **kw): 152 """Invoke reinitialized command `cmdname` with keyword args""" 153 for dirname in INSTALL_DIRECTORY_ATTRS: 154 kw.setdefault(dirname, self.bdist_dir) 155 kw.setdefault('skip_build', self.skip_build) 156 kw.setdefault('dry_run', self.dry_run) 157 cmd = self.reinitialize_command(cmdname, **kw) 158 self.run_command(cmdname) 159 return cmd 160 161 def run(self): 162 # Generate metadata first 163 self.run_command("egg_info") 164 # We run install_lib before install_data, because some data hacks 165 # pull their data path from the install_lib command. 166 log.info("installing library code to %s", self.bdist_dir) 167 instcmd = self.get_finalized_command('install') 168 old_root = instcmd.root 169 instcmd.root = None 170 if self.distribution.has_c_libraries() and not self.skip_build: 171 self.run_command('build_clib') 172 cmd = self.call_command('install_lib', warn_dir=0) 173 instcmd.root = old_root 174 175 all_outputs, ext_outputs = self.get_ext_outputs() 176 self.stubs = [] 177 to_compile = [] 178 for (p, ext_name) in enumerate(ext_outputs): 179 filename, ext = os.path.splitext(ext_name) 180 pyfile = os.path.join(self.bdist_dir, strip_module(filename) + 181 '.py') 182 self.stubs.append(pyfile) 183 log.info("creating stub loader for %s", ext_name) 184 if not self.dry_run: 185 write_stub(os.path.basename(ext_name), pyfile) 186 to_compile.append(pyfile) 187 ext_outputs[p] = ext_name.replace(os.sep, '/') 188 189 if to_compile: 190 cmd.byte_compile(to_compile) 191 if self.distribution.data_files: 192 self.do_install_data() 193 194 # Make the EGG-INFO directory 195 archive_root = self.bdist_dir 196 egg_info = os.path.join(archive_root, 'EGG-INFO') 197 self.mkpath(egg_info) 198 if self.distribution.scripts: 199 script_dir = os.path.join(egg_info, 'scripts') 200 log.info("installing scripts to %s", script_dir) 201 self.call_command('install_scripts', install_dir=script_dir, 202 no_ep=1) 203 204 self.copy_metadata_to(egg_info) 205 native_libs = os.path.join(egg_info, "native_libs.txt") 206 if all_outputs: 207 log.info("writing %s", native_libs) 208 if not self.dry_run: 209 ensure_directory(native_libs) 210 libs_file = open(native_libs, 'wt') 211 libs_file.write('\n'.join(all_outputs)) 212 libs_file.write('\n') 213 libs_file.close() 214 elif os.path.isfile(native_libs): 215 log.info("removing %s", native_libs) 216 if not self.dry_run: 217 os.unlink(native_libs) 218 219 write_safety_flag( 220 os.path.join(archive_root, 'EGG-INFO'), self.zip_safe() 221 ) 222 223 if os.path.exists(os.path.join(self.egg_info, 'depends.txt')): 224 log.warn( 225 "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n" 226 "Use the install_requires/extras_require setup() args instead." 227 ) 228 229 if self.exclude_source_files: 230 self.zap_pyfiles() 231 232 # Make the archive 233 make_zipfile(self.egg_output, archive_root, verbose=self.verbose, 234 dry_run=self.dry_run, mode=self.gen_header()) 235 if not self.keep_temp: 236 remove_tree(self.bdist_dir, dry_run=self.dry_run) 237 238 # Add to 'Distribution.dist_files' so that the "upload" command works 239 getattr(self.distribution, 'dist_files', []).append( 240 ('bdist_egg', get_python_version(), self.egg_output)) 241 242 def zap_pyfiles(self): 243 log.info("Removing .py files from temporary directory") 244 for base, dirs, files in walk_egg(self.bdist_dir): 245 for name in files: 246 path = os.path.join(base, name) 247 248 if name.endswith('.py'): 249 log.debug("Deleting %s", path) 250 os.unlink(path) 251 252 if base.endswith('__pycache__'): 253 path_old = path 254 255 pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc' 256 m = re.match(pattern, name) 257 path_new = os.path.join( 258 base, os.pardir, m.group('name') + '.pyc') 259 log.info( 260 "Renaming file from [%s] to [%s]" 261 % (path_old, path_new)) 262 try: 263 os.remove(path_new) 264 except OSError: 265 pass 266 os.rename(path_old, path_new) 267 268 def zip_safe(self): 269 safe = getattr(self.distribution, 'zip_safe', None) 270 if safe is not None: 271 return safe 272 log.warn("zip_safe flag not set; analyzing archive contents...") 273 return analyze_egg(self.bdist_dir, self.stubs) 274 275 def gen_header(self): 276 epm = EntryPoint.parse_map(self.distribution.entry_points or '') 277 ep = epm.get('setuptools.installation', {}).get('eggsecutable') 278 if ep is None: 279 return 'w' # not an eggsecutable, do it the usual way. 280 281 if not ep.attrs or ep.extras: 282 raise DistutilsSetupError( 283 "eggsecutable entry point (%r) cannot have 'extras' " 284 "or refer to a module" % (ep,) 285 ) 286 287 pyver = sys.version[:3] 288 pkg = ep.module_name 289 full = '.'.join(ep.attrs) 290 base = ep.attrs[0] 291 basename = os.path.basename(self.egg_output) 292 293 header = ( 294 "#!/bin/sh\n" 295 'if [ `basename $0` = "%(basename)s" ]\n' 296 'then exec python%(pyver)s -c "' 297 "import sys, os; sys.path.insert(0, os.path.abspath('$0')); " 298 "from %(pkg)s import %(base)s; sys.exit(%(full)s())" 299 '" "$@"\n' 300 'else\n' 301 ' echo $0 is not the correct name for this egg file.\n' 302 ' echo Please rename it back to %(basename)s and try again.\n' 303 ' exec false\n' 304 'fi\n' 305 ) % locals() 306 307 if not self.dry_run: 308 mkpath(os.path.dirname(self.egg_output), dry_run=self.dry_run) 309 f = open(self.egg_output, 'w') 310 f.write(header) 311 f.close() 312 return 'a' 313 314 def copy_metadata_to(self, target_dir): 315 "Copy metadata (egg info) to the target_dir" 316 # normalize the path (so that a forward-slash in egg_info will 317 # match using startswith below) 318 norm_egg_info = os.path.normpath(self.egg_info) 319 prefix = os.path.join(norm_egg_info, '') 320 for path in self.ei_cmd.filelist.files: 321 if path.startswith(prefix): 322 target = os.path.join(target_dir, path[len(prefix):]) 323 ensure_directory(target) 324 self.copy_file(path, target) 325 326 def get_ext_outputs(self): 327 """Get a list of relative paths to C extensions in the output distro""" 328 329 all_outputs = [] 330 ext_outputs = [] 331 332 paths = {self.bdist_dir: ''} 333 for base, dirs, files in sorted_walk(self.bdist_dir): 334 for filename in files: 335 if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS: 336 all_outputs.append(paths[base] + filename) 337 for filename in dirs: 338 paths[os.path.join(base, filename)] = (paths[base] + 339 filename + '/') 340 341 if self.distribution.has_ext_modules(): 342 build_cmd = self.get_finalized_command('build_ext') 343 for ext in build_cmd.extensions: 344 if isinstance(ext, Library): 345 continue 346 fullname = build_cmd.get_ext_fullname(ext.name) 347 filename = build_cmd.get_ext_filename(fullname) 348 if not os.path.basename(filename).startswith('dl-'): 349 if os.path.exists(os.path.join(self.bdist_dir, filename)): 350 ext_outputs.append(filename) 351 352 return all_outputs, ext_outputs 353 354 355NATIVE_EXTENSIONS = dict.fromkeys('.dll .so .dylib .pyd'.split()) 356 357 358def walk_egg(egg_dir): 359 """Walk an unpacked egg's contents, skipping the metadata directory""" 360 walker = sorted_walk(egg_dir) 361 base, dirs, files = next(walker) 362 if 'EGG-INFO' in dirs: 363 dirs.remove('EGG-INFO') 364 yield base, dirs, files 365 for bdf in walker: 366 yield bdf 367 368 369def analyze_egg(egg_dir, stubs): 370 # check for existing flag in EGG-INFO 371 for flag, fn in safety_flags.items(): 372 if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)): 373 return flag 374 if not can_scan(): 375 return False 376 safe = True 377 for base, dirs, files in walk_egg(egg_dir): 378 for name in files: 379 if name.endswith('.py') or name.endswith('.pyw'): 380 continue 381 elif name.endswith('.pyc') or name.endswith('.pyo'): 382 # always scan, even if we already know we're not safe 383 safe = scan_module(egg_dir, base, name, stubs) and safe 384 return safe 385 386 387def write_safety_flag(egg_dir, safe): 388 # Write or remove zip safety flag file(s) 389 for flag, fn in safety_flags.items(): 390 fn = os.path.join(egg_dir, fn) 391 if os.path.exists(fn): 392 if safe is None or bool(safe) != flag: 393 os.unlink(fn) 394 elif safe is not None and bool(safe) == flag: 395 f = open(fn, 'wt') 396 f.write('\n') 397 f.close() 398 399 400safety_flags = { 401 True: 'zip-safe', 402 False: 'not-zip-safe', 403} 404 405 406def scan_module(egg_dir, base, name, stubs): 407 """Check whether module possibly uses unsafe-for-zipfile stuff""" 408 409 filename = os.path.join(base, name) 410 if filename[:-1] in stubs: 411 return True # Extension module 412 pkg = base[len(egg_dir) + 1:].replace(os.sep, '.') 413 module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0] 414 if sys.version_info < (3, 3): 415 skip = 8 # skip magic & date 416 elif sys.version_info < (3, 7): 417 skip = 12 # skip magic & date & file size 418 else: 419 skip = 16 # skip magic & reserved? & date & file size 420 f = open(filename, 'rb') 421 f.read(skip) 422 code = marshal.load(f) 423 f.close() 424 safe = True 425 symbols = dict.fromkeys(iter_symbols(code)) 426 for bad in ['__file__', '__path__']: 427 if bad in symbols: 428 log.warn("%s: module references %s", module, bad) 429 safe = False 430 if 'inspect' in symbols: 431 for bad in [ 432 'getsource', 'getabsfile', 'getsourcefile', 'getfile' 433 'getsourcelines', 'findsource', 'getcomments', 'getframeinfo', 434 'getinnerframes', 'getouterframes', 'stack', 'trace' 435 ]: 436 if bad in symbols: 437 log.warn("%s: module MAY be using inspect.%s", module, bad) 438 safe = False 439 return safe 440 441 442def iter_symbols(code): 443 """Yield names and strings used by `code` and its nested code objects""" 444 for name in code.co_names: 445 yield name 446 for const in code.co_consts: 447 if isinstance(const, six.string_types): 448 yield const 449 elif isinstance(const, CodeType): 450 for name in iter_symbols(const): 451 yield name 452 453 454def can_scan(): 455 if not sys.platform.startswith('java') and sys.platform != 'cli': 456 # CPython, PyPy, etc. 457 return True 458 log.warn("Unable to analyze compiled code on this platform.") 459 log.warn("Please ask the author to include a 'zip_safe'" 460 " setting (either True or False) in the package's setup.py") 461 462 463# Attribute names of options for commands that might need to be convinced to 464# install to the egg build directory 465 466INSTALL_DIRECTORY_ATTRS = [ 467 'install_lib', 'install_dir', 'install_data', 'install_base' 468] 469 470 471def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True, 472 mode='w'): 473 """Create a zip file from all the files under 'base_dir'. The output 474 zip file will be named 'base_dir' + ".zip". Uses either the "zipfile" 475 Python module (if available) or the InfoZIP "zip" utility (if installed 476 and found on the default search path). If neither tool is available, 477 raises DistutilsExecError. Returns the name of the output zip file. 478 """ 479 import zipfile 480 481 mkpath(os.path.dirname(zip_filename), dry_run=dry_run) 482 log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir) 483 484 def visit(z, dirname, names): 485 for name in names: 486 path = os.path.normpath(os.path.join(dirname, name)) 487 if os.path.isfile(path): 488 p = path[len(base_dir) + 1:] 489 if not dry_run: 490 z.write(path, p) 491 log.debug("adding '%s'", p) 492 493 compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED 494 if not dry_run: 495 z = zipfile.ZipFile(zip_filename, mode, compression=compression) 496 for dirname, dirs, files in sorted_walk(base_dir): 497 visit(z, dirname, files) 498 z.close() 499 else: 500 for dirname, dirs, files in sorted_walk(base_dir): 501 visit(None, dirname, files) 502 return zip_filename 503