1"""setuptools.command.bdist_egg 2 3Build .egg distributions""" 4 5from distutils.dir_util import remove_tree, mkpath 6from distutils import log 7from types import CodeType 8import sys 9import os 10import re 11import textwrap 12import marshal 13 14from pkg_resources import get_build_platform, Distribution 15from setuptools.extension import Library 16from setuptools import Command 17from .._path import ensure_directory 18 19from sysconfig import get_path, get_python_version 20 21 22def _get_purelib(): 23 return get_path("purelib") 24 25 26def strip_module(filename): 27 if '.' in filename: 28 filename = os.path.splitext(filename)[0] 29 if filename.endswith('module'): 30 filename = filename[:-6] 31 return filename 32 33 34def sorted_walk(dir): 35 """Do os.walk in a reproducible way, 36 independent of indeterministic filesystem readdir order 37 """ 38 for base, dirs, files in os.walk(dir): 39 dirs.sort() 40 files.sort() 41 yield base, dirs, files 42 43 44def write_stub(resource, pyfile): 45 _stub_template = textwrap.dedent(""" 46 def __bootstrap__(): 47 global __bootstrap__, __loader__, __file__ 48 import sys, pkg_resources, importlib.util 49 __file__ = pkg_resources.resource_filename(__name__, %r) 50 __loader__ = None; del __bootstrap__, __loader__ 51 spec = importlib.util.spec_from_file_location(__name__,__file__) 52 mod = importlib.util.module_from_spec(spec) 53 spec.loader.exec_module(mod) 54 __bootstrap__() 55 """).lstrip() 56 with open(pyfile, 'w') as f: 57 f.write(_stub_template % resource) 58 59 60class bdist_egg(Command): 61 description = "create an \"egg\" distribution" 62 63 user_options = [ 64 ('bdist-dir=', 'b', 65 "temporary directory for creating the distribution"), 66 ('plat-name=', 'p', "platform name to embed in generated filenames " 67 "(default: %s)" % get_build_platform()), 68 ('exclude-source-files', None, 69 "remove all .py files from the generated egg"), 70 ('keep-temp', 'k', 71 "keep the pseudo-installation tree around after " + 72 "creating the distribution archive"), 73 ('dist-dir=', 'd', 74 "directory to put final built distributions in"), 75 ('skip-build', None, 76 "skip rebuilding everything (for testing/debugging)"), 77 ] 78 79 boolean_options = [ 80 'keep-temp', 'skip-build', 'exclude-source-files' 81 ] 82 83 def initialize_options(self): 84 self.bdist_dir = None 85 self.plat_name = None 86 self.keep_temp = 0 87 self.dist_dir = None 88 self.skip_build = 0 89 self.egg_output = None 90 self.exclude_source_files = None 91 92 def finalize_options(self): 93 ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info") 94 self.egg_info = ei_cmd.egg_info 95 96 if self.bdist_dir is None: 97 bdist_base = self.get_finalized_command('bdist').bdist_base 98 self.bdist_dir = os.path.join(bdist_base, 'egg') 99 100 if self.plat_name is None: 101 self.plat_name = get_build_platform() 102 103 self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) 104 105 if self.egg_output is None: 106 107 # Compute filename of the output egg 108 basename = Distribution( 109 None, None, ei_cmd.egg_name, ei_cmd.egg_version, 110 get_python_version(), 111 self.distribution.has_ext_modules() and self.plat_name 112 ).egg_name() 113 114 self.egg_output = os.path.join(self.dist_dir, basename + '.egg') 115 116 def do_install_data(self): 117 # Hack for packages that install data to install's --install-lib 118 self.get_finalized_command('install').install_lib = self.bdist_dir 119 120 site_packages = os.path.normcase(os.path.realpath(_get_purelib())) 121 old, self.distribution.data_files = self.distribution.data_files, [] 122 123 for item in old: 124 if isinstance(item, tuple) and len(item) == 2: 125 if os.path.isabs(item[0]): 126 realpath = os.path.realpath(item[0]) 127 normalized = os.path.normcase(realpath) 128 if normalized == site_packages or normalized.startswith( 129 site_packages + os.sep 130 ): 131 item = realpath[len(site_packages) + 1:], item[1] 132 # XXX else: raise ??? 133 self.distribution.data_files.append(item) 134 135 try: 136 log.info("installing package data to %s", self.bdist_dir) 137 self.call_command('install_data', force=0, root=None) 138 finally: 139 self.distribution.data_files = old 140 141 def get_outputs(self): 142 return [self.egg_output] 143 144 def call_command(self, cmdname, **kw): 145 """Invoke reinitialized command `cmdname` with keyword args""" 146 for dirname in INSTALL_DIRECTORY_ATTRS: 147 kw.setdefault(dirname, self.bdist_dir) 148 kw.setdefault('skip_build', self.skip_build) 149 kw.setdefault('dry_run', self.dry_run) 150 cmd = self.reinitialize_command(cmdname, **kw) 151 self.run_command(cmdname) 152 return cmd 153 154 def run(self): # noqa: C901 # is too complex (14) # FIXME 155 # Generate metadata first 156 self.run_command("egg_info") 157 # We run install_lib before install_data, because some data hacks 158 # pull their data path from the install_lib command. 159 log.info("installing library code to %s", self.bdist_dir) 160 instcmd = self.get_finalized_command('install') 161 old_root = instcmd.root 162 instcmd.root = None 163 if self.distribution.has_c_libraries() and not self.skip_build: 164 self.run_command('build_clib') 165 cmd = self.call_command('install_lib', warn_dir=0) 166 instcmd.root = old_root 167 168 all_outputs, ext_outputs = self.get_ext_outputs() 169 self.stubs = [] 170 to_compile = [] 171 for (p, ext_name) in enumerate(ext_outputs): 172 filename, ext = os.path.splitext(ext_name) 173 pyfile = os.path.join(self.bdist_dir, strip_module(filename) + 174 '.py') 175 self.stubs.append(pyfile) 176 log.info("creating stub loader for %s", ext_name) 177 if not self.dry_run: 178 write_stub(os.path.basename(ext_name), pyfile) 179 to_compile.append(pyfile) 180 ext_outputs[p] = ext_name.replace(os.sep, '/') 181 182 if to_compile: 183 cmd.byte_compile(to_compile) 184 if self.distribution.data_files: 185 self.do_install_data() 186 187 # Make the EGG-INFO directory 188 archive_root = self.bdist_dir 189 egg_info = os.path.join(archive_root, 'EGG-INFO') 190 self.mkpath(egg_info) 191 if self.distribution.scripts: 192 script_dir = os.path.join(egg_info, 'scripts') 193 log.info("installing scripts to %s", script_dir) 194 self.call_command('install_scripts', install_dir=script_dir, 195 no_ep=1) 196 197 self.copy_metadata_to(egg_info) 198 native_libs = os.path.join(egg_info, "native_libs.txt") 199 if all_outputs: 200 log.info("writing %s", native_libs) 201 if not self.dry_run: 202 ensure_directory(native_libs) 203 libs_file = open(native_libs, 'wt') 204 libs_file.write('\n'.join(all_outputs)) 205 libs_file.write('\n') 206 libs_file.close() 207 elif os.path.isfile(native_libs): 208 log.info("removing %s", native_libs) 209 if not self.dry_run: 210 os.unlink(native_libs) 211 212 write_safety_flag( 213 os.path.join(archive_root, 'EGG-INFO'), self.zip_safe() 214 ) 215 216 if os.path.exists(os.path.join(self.egg_info, 'depends.txt')): 217 log.warn( 218 "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n" 219 "Use the install_requires/extras_require setup() args instead." 220 ) 221 222 if self.exclude_source_files: 223 self.zap_pyfiles() 224 225 # Make the archive 226 make_zipfile(self.egg_output, archive_root, verbose=self.verbose, 227 dry_run=self.dry_run, mode=self.gen_header()) 228 if not self.keep_temp: 229 remove_tree(self.bdist_dir, dry_run=self.dry_run) 230 231 # Add to 'Distribution.dist_files' so that the "upload" command works 232 getattr(self.distribution, 'dist_files', []).append( 233 ('bdist_egg', get_python_version(), self.egg_output)) 234 235 def zap_pyfiles(self): 236 log.info("Removing .py files from temporary directory") 237 for base, dirs, files in walk_egg(self.bdist_dir): 238 for name in files: 239 path = os.path.join(base, name) 240 241 if name.endswith('.py'): 242 log.debug("Deleting %s", path) 243 os.unlink(path) 244 245 if base.endswith('__pycache__'): 246 path_old = path 247 248 pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc' 249 m = re.match(pattern, name) 250 path_new = os.path.join( 251 base, os.pardir, m.group('name') + '.pyc') 252 log.info( 253 "Renaming file from [%s] to [%s]" 254 % (path_old, path_new)) 255 try: 256 os.remove(path_new) 257 except OSError: 258 pass 259 os.rename(path_old, path_new) 260 261 def zip_safe(self): 262 safe = getattr(self.distribution, 'zip_safe', None) 263 if safe is not None: 264 return safe 265 log.warn("zip_safe flag not set; analyzing archive contents...") 266 return analyze_egg(self.bdist_dir, self.stubs) 267 268 def gen_header(self): 269 return 'w' 270 271 def copy_metadata_to(self, target_dir): 272 "Copy metadata (egg info) to the target_dir" 273 # normalize the path (so that a forward-slash in egg_info will 274 # match using startswith below) 275 norm_egg_info = os.path.normpath(self.egg_info) 276 prefix = os.path.join(norm_egg_info, '') 277 for path in self.ei_cmd.filelist.files: 278 if path.startswith(prefix): 279 target = os.path.join(target_dir, path[len(prefix):]) 280 ensure_directory(target) 281 self.copy_file(path, target) 282 283 def get_ext_outputs(self): 284 """Get a list of relative paths to C extensions in the output distro""" 285 286 all_outputs = [] 287 ext_outputs = [] 288 289 paths = {self.bdist_dir: ''} 290 for base, dirs, files in sorted_walk(self.bdist_dir): 291 for filename in files: 292 if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS: 293 all_outputs.append(paths[base] + filename) 294 for filename in dirs: 295 paths[os.path.join(base, filename)] = (paths[base] + 296 filename + '/') 297 298 if self.distribution.has_ext_modules(): 299 build_cmd = self.get_finalized_command('build_ext') 300 for ext in build_cmd.extensions: 301 if isinstance(ext, Library): 302 continue 303 fullname = build_cmd.get_ext_fullname(ext.name) 304 filename = build_cmd.get_ext_filename(fullname) 305 if not os.path.basename(filename).startswith('dl-'): 306 if os.path.exists(os.path.join(self.bdist_dir, filename)): 307 ext_outputs.append(filename) 308 309 return all_outputs, ext_outputs 310 311 312NATIVE_EXTENSIONS = dict.fromkeys('.dll .so .dylib .pyd'.split()) 313 314 315def walk_egg(egg_dir): 316 """Walk an unpacked egg's contents, skipping the metadata directory""" 317 walker = sorted_walk(egg_dir) 318 base, dirs, files = next(walker) 319 if 'EGG-INFO' in dirs: 320 dirs.remove('EGG-INFO') 321 yield base, dirs, files 322 for bdf in walker: 323 yield bdf 324 325 326def analyze_egg(egg_dir, stubs): 327 # check for existing flag in EGG-INFO 328 for flag, fn in safety_flags.items(): 329 if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)): 330 return flag 331 if not can_scan(): 332 return False 333 safe = True 334 for base, dirs, files in walk_egg(egg_dir): 335 for name in files: 336 if name.endswith('.py') or name.endswith('.pyw'): 337 continue 338 elif name.endswith('.pyc') or name.endswith('.pyo'): 339 # always scan, even if we already know we're not safe 340 safe = scan_module(egg_dir, base, name, stubs) and safe 341 return safe 342 343 344def write_safety_flag(egg_dir, safe): 345 # Write or remove zip safety flag file(s) 346 for flag, fn in safety_flags.items(): 347 fn = os.path.join(egg_dir, fn) 348 if os.path.exists(fn): 349 if safe is None or bool(safe) != flag: 350 os.unlink(fn) 351 elif safe is not None and bool(safe) == flag: 352 f = open(fn, 'wt') 353 f.write('\n') 354 f.close() 355 356 357safety_flags = { 358 True: 'zip-safe', 359 False: 'not-zip-safe', 360} 361 362 363def scan_module(egg_dir, base, name, stubs): 364 """Check whether module possibly uses unsafe-for-zipfile stuff""" 365 366 filename = os.path.join(base, name) 367 if filename[:-1] in stubs: 368 return True # Extension module 369 pkg = base[len(egg_dir) + 1:].replace(os.sep, '.') 370 module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0] 371 if sys.version_info < (3, 7): 372 skip = 12 # skip magic & date & file size 373 else: 374 skip = 16 # skip magic & reserved? & date & file size 375 f = open(filename, 'rb') 376 f.read(skip) 377 code = marshal.load(f) 378 f.close() 379 safe = True 380 symbols = dict.fromkeys(iter_symbols(code)) 381 for bad in ['__file__', '__path__']: 382 if bad in symbols: 383 log.warn("%s: module references %s", module, bad) 384 safe = False 385 if 'inspect' in symbols: 386 for bad in [ 387 'getsource', 'getabsfile', 'getsourcefile', 'getfile' 388 'getsourcelines', 'findsource', 'getcomments', 'getframeinfo', 389 'getinnerframes', 'getouterframes', 'stack', 'trace' 390 ]: 391 if bad in symbols: 392 log.warn("%s: module MAY be using inspect.%s", module, bad) 393 safe = False 394 return safe 395 396 397def iter_symbols(code): 398 """Yield names and strings used by `code` and its nested code objects""" 399 for name in code.co_names: 400 yield name 401 for const in code.co_consts: 402 if isinstance(const, str): 403 yield const 404 elif isinstance(const, CodeType): 405 for name in iter_symbols(const): 406 yield name 407 408 409def can_scan(): 410 if not sys.platform.startswith('java') and sys.platform != 'cli': 411 # CPython, PyPy, etc. 412 return True 413 log.warn("Unable to analyze compiled code on this platform.") 414 log.warn("Please ask the author to include a 'zip_safe'" 415 " setting (either True or False) in the package's setup.py") 416 417 418# Attribute names of options for commands that might need to be convinced to 419# install to the egg build directory 420 421INSTALL_DIRECTORY_ATTRS = [ 422 'install_lib', 'install_dir', 'install_data', 'install_base' 423] 424 425 426def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True, 427 mode='w'): 428 """Create a zip file from all the files under 'base_dir'. The output 429 zip file will be named 'base_dir' + ".zip". Uses either the "zipfile" 430 Python module (if available) or the InfoZIP "zip" utility (if installed 431 and found on the default search path). If neither tool is available, 432 raises DistutilsExecError. Returns the name of the output zip file. 433 """ 434 import zipfile 435 436 mkpath(os.path.dirname(zip_filename), dry_run=dry_run) 437 log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir) 438 439 def visit(z, dirname, names): 440 for name in names: 441 path = os.path.normpath(os.path.join(dirname, name)) 442 if os.path.isfile(path): 443 p = path[len(base_dir) + 1:] 444 if not dry_run: 445 z.write(path, p) 446 log.debug("adding '%s'", p) 447 448 compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED 449 if not dry_run: 450 z = zipfile.ZipFile(zip_filename, mode, compression=compression) 451 for dirname, dirs, files in sorted_walk(base_dir): 452 visit(z, dirname, files) 453 z.close() 454 else: 455 for dirname, dirs, files in sorted_walk(base_dir): 456 visit(None, dirname, files) 457 return zip_filename 458