1# Please keep this code python 2.4 compatible and stand alone. 2 3import logging, os, shutil, sys, tempfile, time, urllib2 4import subprocess, re 5from distutils.version import LooseVersion 6 7from autotest_lib.client.common_lib import autotemp, revision_control, utils 8 9_READ_SIZE = 64*1024 10_MAX_PACKAGE_SIZE = 100*1024*1024 11 12 13class Error(Exception): 14 """Local exception to be raised by code in this file.""" 15 16class FetchError(Error): 17 """Failed to fetch a package from any of its listed URLs.""" 18 19 20def _checksum_file(full_path): 21 """@returns The hex checksum of a file given its pathname.""" 22 inputfile = open(full_path, 'rb') 23 try: 24 hex_sum = utils.hash('sha1', inputfile.read()).hexdigest() 25 finally: 26 inputfile.close() 27 return hex_sum 28 29 30def system(commandline): 31 """Same as os.system(commandline) but logs the command first. 32 33 @param commandline: commandline to be called. 34 """ 35 logging.info(commandline) 36 return os.system(commandline) 37 38 39def find_top_of_autotest_tree(): 40 """@returns The full path to the top of the autotest directory tree.""" 41 dirname = os.path.dirname(__file__) 42 autotest_dir = os.path.abspath(os.path.join(dirname, '..')) 43 return autotest_dir 44 45 46class ExternalPackage(object): 47 """ 48 Defines an external package with URLs to fetch its sources from and 49 a build_and_install() method to unpack it, build it and install it 50 beneath our own autotest/site-packages directory. 51 52 Base Class. Subclass this to define packages. 53 Note: Unless your subclass has a specific reason to, it should not 54 re-install the package every time build_externals is invoked, as this 55 happens periodically through the scheduler. To avoid doing so the is_needed 56 method needs to return an appropriate value. 57 58 Attributes: 59 @attribute urls - A tuple of URLs to try fetching the package from. 60 @attribute local_filename - A local filename to use when saving the 61 fetched package. 62 @attribute dist_name - The name of the Python distribution. For example, 63 the package MySQLdb is included in the distribution named 64 MySQL-python. This is generally the PyPI name. Defaults to the 65 name part of the local_filename. 66 @attribute hex_sum - The hex digest (currently SHA1) of this package 67 to be used to verify its contents. 68 @attribute module_name - The installed python module name to be used for 69 for a version check. Defaults to the lower case class name with 70 the word Package stripped off. 71 @attribute extracted_package_path - The path to package directory after 72 extracting. 73 @attribute version - The desired minimum package version. 74 @attribute os_requirements - A dictionary mapping pathname tuples on the 75 the OS distribution to a likely name of a package the user 76 needs to install on their system in order to get this file. 77 One of the files in the tuple must exist. 78 @attribute name - Read only, the printable name of the package. 79 @attribute subclasses - This class attribute holds a list of all defined 80 subclasses. It is constructed dynamically using the metaclass. 81 """ 82 # Modules that are meant to be installed in system directory, rather than 83 # autotest/site-packages. These modules should be skipped if the module 84 # is already installed in system directory. This prevents an older version 85 # of the module from being installed in system directory. 86 SYSTEM_MODULES = ['setuptools'] 87 88 subclasses = [] 89 urls = () 90 local_filename = None 91 dist_name = None 92 hex_sum = None 93 module_name = None 94 version = None 95 os_requirements = None 96 97 98 class __metaclass__(type): 99 """Any time a subclass is defined, add it to our list.""" 100 def __init__(mcs, name, bases, dict): 101 if name != 'ExternalPackage' and not name.startswith('_'): 102 mcs.subclasses.append(mcs) 103 104 105 def __init__(self): 106 self.verified_package = '' 107 if not self.module_name: 108 self.module_name = self.name.lower() 109 if not self.dist_name and self.local_filename: 110 self.dist_name = self.local_filename[:self.local_filename.rindex('-')] 111 self.installed_version = '' 112 113 114 @property 115 def extracted_package_path(self): 116 """Return the package path after extracting. 117 118 If the package has assigned its own extracted_package_path, use it. 119 Or use part of its local_filename as the extracting path. 120 """ 121 return self.local_filename[:-len(self._get_extension( 122 self.local_filename))] 123 124 125 @property 126 def name(self): 127 """Return the class name with any trailing 'Package' stripped off.""" 128 class_name = self.__class__.__name__ 129 if class_name.endswith('Package'): 130 return class_name[:-len('Package')] 131 return class_name 132 133 134 def is_needed(self, install_dir): 135 """ 136 Check to see if we need to reinstall a package. This is contingent on: 137 1. Module name: If the name of the module is different from the package, 138 the class that installs it needs to specify a module_name string, 139 so we can try importing the module. 140 141 2. Installed version: If the module doesn't contain a __version__ the 142 class that installs it needs to override the 143 _get_installed_version_from_module method to return an appropriate 144 version string. 145 146 3. Version/Minimum version: The class that installs the package should 147 contain a version string, and an optional minimum version string. 148 149 4. install_dir: If the module exists in a different directory, e.g., 150 /usr/lib/python2.7/dist-packages/, the module will be forced to be 151 installed in install_dir. 152 153 @param install_dir: install directory. 154 @returns True if self.module_name needs to be built and installed. 155 """ 156 if not self.module_name or not self.version: 157 logging.warning('version and module_name required for ' 158 'is_needed() check to work.') 159 return True 160 try: 161 module = __import__(self.module_name) 162 except ImportError, e: 163 logging.info("%s isn't present. Will install.", self.module_name) 164 return True 165 if (not module.__file__.startswith(install_dir) and 166 not self.module_name in self.SYSTEM_MODULES): 167 logging.info('Module %s is installed in %s, rather than %s. The ' 168 'module will be forced to be installed in %s.', 169 self.module_name, module.__file__, install_dir, 170 install_dir) 171 return True 172 self.installed_version = self._get_installed_version_from_module(module) 173 if not self.installed_version: 174 return True 175 176 logging.info('imported %s version %s.', self.module_name, 177 self.installed_version) 178 if hasattr(self, 'minimum_version'): 179 return LooseVersion(self.minimum_version) > LooseVersion( 180 self.installed_version) 181 else: 182 return LooseVersion(self.version) > LooseVersion( 183 self.installed_version) 184 185 186 def _get_installed_version_from_module(self, module): 187 """Ask our module its version string and return it or '' if unknown.""" 188 try: 189 return module.__version__ 190 except AttributeError: 191 logging.error('could not get version from %s', module) 192 return '' 193 194 195 def _build_and_install(self, install_dir): 196 """Subclasses MUST provide their own implementation.""" 197 raise NotImplementedError 198 199 200 def _build_and_install_current_dir(self, install_dir): 201 """ 202 Subclasses that use _build_and_install_from_package() MUST provide 203 their own implementation of this method. 204 """ 205 raise NotImplementedError 206 207 208 def build_and_install(self, install_dir): 209 """ 210 Builds and installs the package. It must have been fetched already. 211 212 @param install_dir - The package installation directory. If it does 213 not exist it will be created. 214 """ 215 if not self.verified_package: 216 raise Error('Must call fetch() first. - %s' % self.name) 217 self._check_os_requirements() 218 return self._build_and_install(install_dir) 219 220 221 def _check_os_requirements(self): 222 if not self.os_requirements: 223 return 224 failed = False 225 for file_names, package_name in self.os_requirements.iteritems(): 226 if not any(os.path.exists(file_name) for file_name in file_names): 227 failed = True 228 logging.error('Can\'t find %s, %s probably needs it.', 229 ' or '.join(file_names), self.name) 230 logging.error('Perhaps you need to install something similar ' 231 'to the %s package for OS first.', package_name) 232 if failed: 233 raise Error('Missing OS requirements for %s. (see above)' % 234 self.name) 235 236 237 def _build_and_install_current_dir_setup_py(self, install_dir): 238 """For use as a _build_and_install_current_dir implementation.""" 239 egg_path = self._build_egg_using_setup_py(setup_py='setup.py') 240 if not egg_path: 241 return False 242 return self._install_from_egg(install_dir, egg_path) 243 244 245 def _build_and_install_current_dir_setupegg_py(self, install_dir): 246 """For use as a _build_and_install_current_dir implementation.""" 247 egg_path = self._build_egg_using_setup_py(setup_py='setupegg.py') 248 if not egg_path: 249 return False 250 return self._install_from_egg(install_dir, egg_path) 251 252 253 def _build_and_install_current_dir_noegg(self, install_dir): 254 if not self._build_using_setup_py(): 255 return False 256 return self._install_using_setup_py_and_rsync(install_dir) 257 258 259 def _get_extension(self, package): 260 """Get extension of package.""" 261 valid_package_extensions = ['.tar.gz', '.tar.bz2', '.zip'] 262 extension = None 263 264 for ext in valid_package_extensions: 265 if package.endswith(ext): 266 extension = ext 267 break 268 269 if not extension: 270 raise Error('Unexpected package file extension on %s' % package) 271 272 return extension 273 274 275 def _build_and_install_from_package(self, install_dir): 276 """ 277 This method may be used as a _build_and_install() implementation 278 for subclasses if they implement _build_and_install_current_dir(). 279 280 Extracts the .tar.gz file, chdirs into the extracted directory 281 (which is assumed to match the tar filename) and calls 282 _build_and_isntall_current_dir from there. 283 284 Afterwards the build (regardless of failure) extracted .tar.gz 285 directory is cleaned up. 286 287 @returns True on success, False otherwise. 288 289 @raises OSError If the expected extraction directory does not exist. 290 """ 291 self._extract_compressed_package() 292 extension = self._get_extension(self.verified_package) 293 os.chdir(os.path.dirname(self.verified_package)) 294 os.chdir(self.extracted_package_path) 295 extracted_dir = os.getcwd() 296 try: 297 return self._build_and_install_current_dir(install_dir) 298 finally: 299 os.chdir(os.path.join(extracted_dir, '..')) 300 shutil.rmtree(extracted_dir) 301 302 303 def _extract_compressed_package(self): 304 """Extract the fetched compressed .tar or .zip within its directory.""" 305 if not self.verified_package: 306 raise Error('Package must have been fetched first.') 307 os.chdir(os.path.dirname(self.verified_package)) 308 if self.verified_package.endswith('gz'): 309 status = system("tar -xzf '%s'" % self.verified_package) 310 elif self.verified_package.endswith('bz2'): 311 status = system("tar -xjf '%s'" % self.verified_package) 312 elif self.verified_package.endswith('zip'): 313 status = system("unzip '%s'" % self.verified_package) 314 else: 315 raise Error('Unknown compression suffix on %s.' % 316 self.verified_package) 317 if status: 318 raise Error('tar failed with %s' % (status,)) 319 320 321 def _build_using_setup_py(self, setup_py='setup.py'): 322 """ 323 Assuming the cwd is the extracted python package, execute a simple 324 python setup.py build. 325 326 @param setup_py - The name of the setup.py file to execute. 327 328 @returns True on success, False otherwise. 329 """ 330 if not os.path.exists(setup_py): 331 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 332 status = system("'%s' %s build" % (sys.executable, setup_py)) 333 if status: 334 logging.error('%s build failed.', self.name) 335 return False 336 return True 337 338 339 def _build_egg_using_setup_py(self, setup_py='setup.py'): 340 """ 341 Assuming the cwd is the extracted python package, execute a simple 342 python setup.py bdist_egg. 343 344 @param setup_py - The name of the setup.py file to execute. 345 346 @returns The relative path to the resulting egg file or '' on failure. 347 """ 348 if not os.path.exists(setup_py): 349 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 350 egg_subdir = 'dist' 351 if os.path.isdir(egg_subdir): 352 shutil.rmtree(egg_subdir) 353 status = system("'%s' %s bdist_egg" % (sys.executable, setup_py)) 354 if status: 355 logging.error('bdist_egg of setuptools failed.') 356 return '' 357 # I've never seen a bdist_egg lay multiple .egg files. 358 for filename in os.listdir(egg_subdir): 359 if filename.endswith('.egg'): 360 return os.path.join(egg_subdir, filename) 361 362 363 def _install_from_egg(self, install_dir, egg_path): 364 """ 365 Install a module from an egg file by unzipping the necessary parts 366 into install_dir. 367 368 @param install_dir - The installation directory. 369 @param egg_path - The pathname of the egg file. 370 """ 371 status = system("unzip -q -o -d '%s' '%s'" % (install_dir, egg_path)) 372 if status: 373 logging.error('unzip of %s failed', egg_path) 374 return False 375 egg_info_dir = os.path.join(install_dir, 'EGG-INFO') 376 if os.path.isdir(egg_info_dir): 377 egg_info_new_path = self._get_egg_info_path(install_dir) 378 if egg_info_new_path: 379 if os.path.exists(egg_info_new_path): 380 shutil.rmtree(egg_info_new_path) 381 os.rename(egg_info_dir, egg_info_new_path) 382 else: 383 shutil.rmtree(egg_info_dir) 384 return True 385 386 387 def _get_egg_info_path(self, install_dir): 388 """Get egg-info path for this package. 389 390 Example path: install_dir/MySQL_python-1.2.3.egg-info 391 392 """ 393 if self.dist_name: 394 egg_info_name_part = self.dist_name.replace('-', '_') 395 if self.version: 396 egg_info_filename = '%s-%s.egg-info' % (egg_info_name_part, 397 self.version) 398 else: 399 egg_info_filename = '%s.egg-info' % (egg_info_name_part,) 400 return os.path.join(install_dir, egg_info_filename) 401 else: 402 return None 403 404 405 def _get_temp_dir(self): 406 return tempfile.mkdtemp(dir='/var/tmp') 407 408 409 def _site_packages_path(self, temp_dir): 410 # This makes assumptions about what python setup.py install 411 # does when given a prefix. Is this always correct? 412 python_xy = 'python%s' % sys.version[:3] 413 return os.path.join(temp_dir, 'lib', python_xy, 'site-packages') 414 415 416 def _rsync (self, temp_site_dir, install_dir): 417 """Rsync contents. """ 418 status = system("rsync -r '%s/' '%s/'" % 419 (os.path.normpath(temp_site_dir), 420 os.path.normpath(install_dir))) 421 if status: 422 logging.error('%s rsync to install_dir failed.', self.name) 423 return False 424 return True 425 426 427 def _install_using_setup_py_and_rsync(self, install_dir, 428 setup_py='setup.py', 429 temp_dir=None): 430 """ 431 Assuming the cwd is the extracted python package, execute a simple: 432 433 python setup.py install --prefix=BLA 434 435 BLA will be a temporary directory that everything installed will 436 be picked out of and rsynced to the appropriate place under 437 install_dir afterwards. 438 439 Afterwards, it deconstructs the extra lib/pythonX.Y/site-packages/ 440 directory tree that setuptools created and moves all installed 441 site-packages directly up into install_dir itself. 442 443 @param install_dir the directory for the install to happen under. 444 @param setup_py - The name of the setup.py file to execute. 445 446 @returns True on success, False otherwise. 447 """ 448 if not os.path.exists(setup_py): 449 raise Error('%s does not exist in %s' % (setup_py, os.getcwd())) 450 451 if temp_dir is None: 452 temp_dir = self._get_temp_dir() 453 454 try: 455 status = system("'%s' %s install --no-compile --prefix='%s'" 456 % (sys.executable, setup_py, temp_dir)) 457 if status: 458 logging.error('%s install failed.', self.name) 459 return False 460 461 if os.path.isdir(os.path.join(temp_dir, 'lib')): 462 # NOTE: This ignores anything outside of the lib/ dir that 463 # was installed. 464 temp_site_dir = self._site_packages_path(temp_dir) 465 else: 466 temp_site_dir = temp_dir 467 468 return self._rsync(temp_site_dir, install_dir) 469 finally: 470 shutil.rmtree(temp_dir) 471 472 473 474 def _build_using_make(self, install_dir): 475 """Build the current package using configure/make. 476 477 @returns True on success, False otherwise. 478 """ 479 install_prefix = os.path.join(install_dir, 'usr', 'local') 480 status = system('./configure --prefix=%s' % install_prefix) 481 if status: 482 logging.error('./configure failed for %s', self.name) 483 return False 484 status = system('make') 485 if status: 486 logging.error('make failed for %s', self.name) 487 return False 488 status = system('make check') 489 if status: 490 logging.error('make check failed for %s', self.name) 491 return False 492 return True 493 494 495 def _install_using_make(self): 496 """Install the current package using make install. 497 498 Assumes the install path was set up while running ./configure (in 499 _build_using_make()). 500 501 @returns True on success, False otherwise. 502 """ 503 status = system('make install') 504 return status == 0 505 506 507 def fetch(self, dest_dir): 508 """ 509 Fetch the package from one its URLs and save it in dest_dir. 510 511 If the the package already exists in dest_dir and the checksum 512 matches this code will not fetch it again. 513 514 Sets the 'verified_package' attribute with the destination pathname. 515 516 @param dest_dir - The destination directory to save the local file. 517 If it does not exist it will be created. 518 519 @returns A boolean indicating if we the package is now in dest_dir. 520 @raises FetchError - When something unexpected happens. 521 """ 522 if not os.path.exists(dest_dir): 523 os.makedirs(dest_dir) 524 local_path = os.path.join(dest_dir, self.local_filename) 525 526 # If the package exists, verify its checksum and be happy if it is good. 527 if os.path.exists(local_path): 528 actual_hex_sum = _checksum_file(local_path) 529 if self.hex_sum == actual_hex_sum: 530 logging.info('Good checksum for existing %s package.', 531 self.name) 532 self.verified_package = local_path 533 return True 534 logging.warning('Bad checksum for existing %s package. ' 535 'Re-downloading', self.name) 536 os.rename(local_path, local_path + '.wrong-checksum') 537 538 # Download the package from one of its urls, rejecting any if the 539 # checksum does not match. 540 for url in self.urls: 541 logging.info('Fetching %s', url) 542 try: 543 url_file = urllib2.urlopen(url) 544 except (urllib2.URLError, EnvironmentError): 545 logging.warning('Could not fetch %s package from %s.', 546 self.name, url) 547 continue 548 549 data_length = int(url_file.info().get('Content-Length', 550 _MAX_PACKAGE_SIZE)) 551 if data_length <= 0 or data_length > _MAX_PACKAGE_SIZE: 552 raise FetchError('%s from %s fails Content-Length %d ' 553 'sanity check.' % (self.name, url, 554 data_length)) 555 checksum = utils.hash('sha1') 556 total_read = 0 557 output = open(local_path, 'wb') 558 try: 559 while total_read < data_length: 560 data = url_file.read(_READ_SIZE) 561 if not data: 562 break 563 output.write(data) 564 checksum.update(data) 565 total_read += len(data) 566 finally: 567 output.close() 568 if self.hex_sum != checksum.hexdigest(): 569 logging.warning('Bad checksum for %s fetched from %s.', 570 self.name, url) 571 logging.warning('Got %s', checksum.hexdigest()) 572 logging.warning('Expected %s', self.hex_sum) 573 os.unlink(local_path) 574 continue 575 logging.info('Good checksum.') 576 self.verified_package = local_path 577 return True 578 else: 579 return False 580 581 582# NOTE: This class definition must come -before- all other ExternalPackage 583# classes that need to use this version of setuptools so that is is inserted 584# into the ExternalPackage.subclasses list before them. 585class SetuptoolsPackage(ExternalPackage): 586 """setuptools package""" 587 # For all known setuptools releases a string compare works for the 588 # version string. Hopefully they never release a 0.10. (Their own 589 # version comparison code would break if they did.) 590 # Any system with setuptools > 18.0.1 is fine. If none installed, then 591 # try to install the latest found on the upstream. 592 minimum_version = '18.0.1' 593 version = '18.0.1' 594 urls = ('http://pypi.python.org/packages/source/s/setuptools/' 595 'setuptools-%s.tar.gz' % (version,),) 596 local_filename = 'setuptools-%s.tar.gz' % version 597 hex_sum = 'ebc4fe81b7f6d61d923d9519f589903824044f52' 598 599 SUDO_SLEEP_DELAY = 15 600 601 602 def _build_and_install(self, install_dir): 603 """Install setuptools on the system.""" 604 logging.info('NOTE: setuptools install does not use install_dir.') 605 return self._build_and_install_from_package(install_dir) 606 607 608 def _build_and_install_current_dir(self, install_dir): 609 egg_path = self._build_egg_using_setup_py() 610 if not egg_path: 611 return False 612 613 print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n' 614 print 'About to run sudo to install setuptools', self.version 615 print 'on your system for use by', sys.executable, '\n' 616 print '!! ^C within', self.SUDO_SLEEP_DELAY, 'seconds to abort.\n' 617 time.sleep(self.SUDO_SLEEP_DELAY) 618 619 # Copy the egg to the local filesystem /var/tmp so that root can 620 # access it properly (avoid NFS squashroot issues). 621 temp_dir = self._get_temp_dir() 622 try: 623 shutil.copy(egg_path, temp_dir) 624 egg_name = os.path.split(egg_path)[1] 625 temp_egg = os.path.join(temp_dir, egg_name) 626 p = subprocess.Popen(['sudo', '/bin/sh', temp_egg], 627 stdout=subprocess.PIPE) 628 regex = re.compile('Copying (.*?) to (.*?)\n') 629 match = regex.search(p.communicate()[0]) 630 status = p.wait() 631 632 if match: 633 compiled = os.path.join(match.group(2), match.group(1)) 634 os.system("sudo chmod a+r '%s'" % compiled) 635 finally: 636 shutil.rmtree(temp_dir) 637 638 if status: 639 logging.error('install of setuptools from egg failed.') 640 return False 641 return True 642 643 644class MySQLdbPackage(ExternalPackage): 645 """mysql package, used in scheduler.""" 646 module_name = 'MySQLdb' 647 version = '1.2.3' 648 urls = ('http://downloads.sourceforge.net/project/mysql-python/' 649 'mysql-python/%(version)s/MySQL-python-%(version)s.tar.gz' 650 % dict(version=version),) 651 local_filename = 'MySQL-python-%s.tar.gz' % version 652 hex_sum = '3511bb8c57c6016eeafa531d5c3ea4b548915e3c' 653 654 _build_and_install_current_dir = ( 655 ExternalPackage._build_and_install_current_dir_setup_py) 656 657 658 def _build_and_install(self, install_dir): 659 if not os.path.exists('/usr/bin/mysql_config'): 660 error_msg = ('You need to install /usr/bin/mysql_config.\n' 661 'On Ubuntu or Debian based systems use this: ' 662 'sudo apt-get install libmysqlclient15-dev') 663 logging.error(error_msg) 664 return False, error_msg 665 return self._build_and_install_from_package(install_dir) 666 667 668class DjangoPackage(ExternalPackage): 669 """django package.""" 670 version = '1.5.1' 671 local_filename = 'Django-%s.tar.gz' % version 672 urls = ('http://www.djangoproject.com/download/%s/tarball/' % version,) 673 hex_sum = '0ab97b90c4c79636e56337f426f1e875faccbba1' 674 675 _build_and_install = ExternalPackage._build_and_install_from_package 676 _build_and_install_current_dir = ( 677 ExternalPackage._build_and_install_current_dir_noegg) 678 679 680 def _get_installed_version_from_module(self, module): 681 try: 682 return module.get_version().split()[0] 683 except AttributeError: 684 return '0.9.6' 685 686 687 688class NumpyPackage(ExternalPackage): 689 """numpy package, required by matploglib.""" 690 version = '1.7.0' 691 local_filename = 'numpy-%s.tar.gz' % version 692 urls = ('http://downloads.sourceforge.net/project/numpy/NumPy/%(version)s/' 693 'numpy-%(version)s.tar.gz' % dict(version=version),) 694 hex_sum = 'ba328985f20390b0f969a5be2a6e1141d5752cf9' 695 696 _build_and_install = ExternalPackage._build_and_install_from_package 697 _build_and_install_current_dir = ( 698 ExternalPackage._build_and_install_current_dir_setupegg_py) 699 700 701class MatplotlibPackage(ExternalPackage): 702 """ 703 matplotlib package 704 705 This requires numpy so it must be declared after numpy to guarantee that 706 it is already installed. 707 """ 708 version = '0.98.5.3' 709 short_version = '0.98.5' 710 local_filename = 'matplotlib-%s.tar.gz' % version 711 urls = ('http://downloads.sourceforge.net/project/matplotlib/matplotlib/' 712 'matplotlib-%s/matplotlib-%s.tar.gz' % (short_version, version),) 713 hex_sum = '2f6c894cf407192b3b60351bcc6468c0385d47b6' 714 os_requirements = {('/usr/include/freetype2/ft2build.h', 715 '/usr/include/ft2build.h'): 'libfreetype6-dev', 716 ('/usr/include/png.h'): 'libpng12-dev'} 717 718 _build_and_install = ExternalPackage._build_and_install_from_package 719 _build_and_install_current_dir = ( 720 ExternalPackage._build_and_install_current_dir_setupegg_py) 721 722 723class JsonRPCLib(ExternalPackage): 724 """jsonrpclib package""" 725 version = '0.1.3' 726 module_name = 'jsonrpclib' 727 local_filename = '%s-%s.tar.gz' % (module_name, version) 728 urls = ('http://pypi.python.org/packages/source/j/%s/%s' % 729 (module_name, local_filename), ) 730 hex_sum = '431714ed19ab677f641ce5d678a6a95016f5c452' 731 732 def _get_installed_version_from_module(self, module): 733 # jsonrpclib doesn't contain a proper version 734 return self.version 735 736 _build_and_install = ExternalPackage._build_and_install_from_package 737 _build_and_install_current_dir = ( 738 ExternalPackage._build_and_install_current_dir_noegg) 739 740 741class GwtPackage(ExternalPackage): 742 """Fetch and extract a local copy of GWT used to build the frontend.""" 743 744 version = '2.3.0' 745 local_filename = 'gwt-%s.zip' % version 746 urls = ('https://storage.googleapis.com/google-code-archive-downloads/' 747 'v2/code.google.com/google-web-toolkit/' + local_filename,) 748 hex_sum = 'd51fce9166e6b31349659ffca89baf93e39bc84b' 749 name = 'gwt' 750 about_filename = 'about.txt' 751 module_name = None # Not a Python module. 752 753 754 def is_needed(self, install_dir): 755 gwt_dir = os.path.join(install_dir, self.name) 756 about_file = os.path.join(install_dir, self.name, self.about_filename) 757 758 if not os.path.exists(gwt_dir) or not os.path.exists(about_file): 759 logging.info('gwt not installed for autotest') 760 return True 761 762 f = open(about_file, 'r') 763 version_line = f.readline() 764 f.close() 765 766 match = re.match(r'Google Web Toolkit (.*)', version_line) 767 if not match: 768 logging.info('did not find gwt version') 769 return True 770 771 logging.info('found gwt version %s', match.group(1)) 772 return match.group(1) != self.version 773 774 775 def _build_and_install(self, install_dir): 776 os.chdir(install_dir) 777 self._extract_compressed_package() 778 extracted_dir = self.local_filename[:-len('.zip')] 779 target_dir = os.path.join(install_dir, self.name) 780 if os.path.exists(target_dir): 781 shutil.rmtree(target_dir) 782 os.rename(extracted_dir, target_dir) 783 return True 784 785 786class GdataPackage(ExternalPackage): 787 """ 788 Pulls the GData library, giving us an API to query tracker. 789 """ 790 version = '2.0.18' 791 local_filename = 'gdata-%s.zip' % version 792 urls = ('https://github.com/google/gdata-python-client/' + 793 'archive/master.zip',) 794 hex_sum = '893f9c9f627ef92afe8f3f066311d9b3748f1732' 795 extracted_package_path = 'gdata-python-client-master' 796 797 _build_and_install = ExternalPackage._build_and_install_from_package 798 _build_and_install_current_dir = ( 799 ExternalPackage._build_and_install_current_dir_noegg) 800 801 def _get_installed_version_from_module(self, module): 802 # gdata doesn't contain a proper version 803 return self.version 804 805 806class PyudevPackage(ExternalPackage): 807 """ 808 pyudev module 809 810 Used in unittests. 811 """ 812 version = '0.16.1' 813 url_filename = 'pyudev-%s.tar.gz' % version 814 local_filename = url_filename 815 urls = ('http://pypi.python.org/packages/source/p/pyudev/%s' % ( 816 url_filename),) 817 hex_sum = 'b36bc5c553ce9b56d32a5e45063a2c88156771c0' 818 819 _build_and_install = ExternalPackage._build_and_install_from_package 820 _build_and_install_current_dir = ( 821 ExternalPackage._build_and_install_current_dir_setup_py) 822 823 824class PyMoxPackage(ExternalPackage): 825 """ 826 mox module 827 828 Used in unittests. 829 """ 830 module_name = 'mox' 831 version = '0.5.3' 832 url_filename = 'mox-%s.tar.gz' % version 833 local_filename = url_filename 834 urls = ('http://pypi.python.org/packages/source/m/mox/%s' % ( 835 url_filename),) 836 hex_sum = '1c502d2c0a8aefbba2c7f385a83d33e7d822452a' 837 838 _build_and_install = ExternalPackage._build_and_install_from_package 839 _build_and_install_current_dir = ( 840 ExternalPackage._build_and_install_current_dir_noegg) 841 842 def _get_installed_version_from_module(self, module): 843 # mox doesn't contain a proper version 844 return self.version 845 846 847class MockPackage(ExternalPackage): 848 """ 849 mock module 850 851 Used in unittests. 852 """ 853 module_name = 'mock' 854 version = '2.0.0' 855 url_filename = 'mock-%s.tar.gz' % version 856 local_filename = url_filename 857 urls = ('http://pypi.python.org/packages/source/m/mock/%s' % ( 858 url_filename),) 859 hex_sum = '397ed52eb2d8d4b326bc3fa6b38adda5f0b090d3' 860 861 _build_and_install = ExternalPackage._build_and_install_from_package 862 _build_and_install_current_dir = ( 863 ExternalPackage._build_and_install_current_dir_noegg) 864 865 866class PySeleniumPackage(ExternalPackage): 867 """ 868 selenium module 869 870 Used in wifi_interop suite. 871 """ 872 module_name = 'selenium' 873 version = '2.37.2' 874 url_filename = 'selenium-%s.tar.gz' % version 875 local_filename = url_filename 876 urls = ('https://pypi.python.org/packages/source/s/selenium/%s' % ( 877 url_filename),) 878 hex_sum = '66946d5349e36d946daaad625c83c30c11609e36' 879 880 _build_and_install = ExternalPackage._build_and_install_from_package 881 _build_and_install_current_dir = ( 882 ExternalPackage._build_and_install_current_dir_setup_py) 883 884 885class FaultHandlerPackage(ExternalPackage): 886 """ 887 faulthandler module 888 """ 889 module_name = 'faulthandler' 890 version = '2.3' 891 url_filename = '%s-%s.tar.gz' % (module_name, version) 892 local_filename = url_filename 893 urls = ('http://pypi.python.org/packages/source/f/faulthandler/%s' % 894 (url_filename),) 895 hex_sum = 'efb30c068414fba9df892e48fcf86170cbf53589' 896 897 _build_and_install = ExternalPackage._build_and_install_from_package 898 _build_and_install_current_dir = ( 899 ExternalPackage._build_and_install_current_dir_noegg) 900 901 902class PsutilPackage(ExternalPackage): 903 """ 904 psutil module 905 """ 906 module_name = 'psutil' 907 version = '2.1.1' 908 url_filename = '%s-%s.tar.gz' % (module_name, version) 909 local_filename = url_filename 910 urls = ('http://pypi.python.org/packages/source/p/psutil/%s' % 911 (url_filename),) 912 hex_sum = '0c20a20ed316e69f2b0881530439213988229916' 913 914 _build_and_install = ExternalPackage._build_and_install_from_package 915 _build_and_install_current_dir = ( 916 ExternalPackage._build_and_install_current_dir_setup_py) 917 918 919class ElasticSearchPackage(ExternalPackage): 920 """elasticsearch-py package.""" 921 version = '1.6.0' 922 url_filename = 'elasticsearch-%s.tar.gz' % version 923 local_filename = url_filename 924 urls = ('https://pypi.python.org/packages/source/e/elasticsearch/%s' % 925 (url_filename),) 926 hex_sum = '3e676c96f47935b1f52df82df3969564bd356b1c' 927 _build_and_install = ExternalPackage._build_and_install_from_package 928 _build_and_install_current_dir = ( 929 ExternalPackage._build_and_install_current_dir_setup_py) 930 931 def _get_installed_version_from_module(self, module): 932 # Elastic's version format is like tuple (1, 6, 0), which needs to be 933 # transferred to 1.6.0. 934 try: 935 return '.'.join(str(i) for i in module.__version__) 936 except: 937 return self.version 938 939 940class Urllib3Package(ExternalPackage): 941 """elasticsearch-py package.""" 942 version = '1.9' 943 url_filename = 'urllib3-%s.tar.gz' % version 944 local_filename = url_filename 945 urls = ('https://pypi.python.org/packages/source/u/urllib3/%s' % 946 (url_filename),) 947 hex_sum = '9522197efb2a2b49ce804de3a515f06d97b6602f' 948 _build_and_install = ExternalPackage._build_and_install_from_package 949 _build_and_install_current_dir = ( 950 ExternalPackage._build_and_install_current_dir_setup_py) 951 952class ImagingLibraryPackage(ExternalPackage): 953 """Python Imaging Library (PIL).""" 954 version = '1.1.7' 955 url_filename = 'Imaging-%s.tar.gz' % version 956 local_filename = url_filename 957 urls = ('http://commondatastorage.googleapis.com/chromeos-mirror/gentoo/' 958 'distfiles/%s' % url_filename,) 959 hex_sum = '76c37504251171fda8da8e63ecb8bc42a69a5c81' 960 961 def _build_and_install(self, install_dir): 962 #The path of zlib library might be different from what PIL setup.py is 963 #expected. Following change does the best attempt to link the library 964 #to a path PIL setup.py will try. 965 libz_possible_path = '/usr/lib/x86_64-linux-gnu/libz.so' 966 libz_expected_path = '/usr/lib/libz.so' 967 if (os.path.exists(libz_possible_path) and 968 not os.path.exists(libz_expected_path)): 969 utils.run('sudo ln -s %s %s' % 970 (libz_possible_path, libz_expected_path)) 971 return self._build_and_install_from_package(install_dir) 972 973 _build_and_install_current_dir = ( 974 ExternalPackage._build_and_install_current_dir_noegg) 975 976 977class AstroidPackage(ExternalPackage): 978 """astroid package.""" 979 version = '1.0.0' 980 url_filename = 'astroid-%s.tar.gz' % version 981 local_filename = url_filename 982 #md5=e74430dfbbe09cd18ef75bd76f95425a 983 urls = ('https://pypi.python.org/packages/15/ef/' 984 '1c01161c40ce08451254125935c5bca85b08913e610a4708760ee1432fa8/%s' % 985 (url_filename),) 986 hex_sum = '2ebba76d115cb8a2d84d8777d8535ddac86daaa6' 987 _build_and_install = ExternalPackage._build_and_install_from_package 988 _build_and_install_current_dir = ( 989 ExternalPackage._build_and_install_current_dir_setup_py) 990 991 992class LogilabCommonPackage(ExternalPackage): 993 """logilab-common package.""" 994 version = '1.2.2' 995 module_name = 'logilab' 996 url_filename = 'logilab-common-%s.tar.gz' % version 997 local_filename = url_filename 998 #md5=daa7b20c8374ff5f525882cf67e258c0 999 urls = ('https://pypi.python.org/packages/63/5b/' 1000 'd4d93ad9e683a06354bc5893194514fbf5d05ef86b06b0285762c3724509/%s' % 1001 (url_filename),) 1002 hex_sum = 'ecad2d10c31dcf183c8bed87b6ec35e7ed397d27' 1003 _build_and_install = ExternalPackage._build_and_install_from_package 1004 _build_and_install_current_dir = ( 1005 ExternalPackage._build_and_install_current_dir_setup_py) 1006 1007 1008class PyLintPackage(ExternalPackage): 1009 """pylint package.""" 1010 version = '1.1.0' 1011 url_filename = 'pylint-%s.tar.gz' % version 1012 local_filename = url_filename 1013 #md5=017299b5911838a9347a71de5f946afc 1014 urls = ('https://pypi.python.org/packages/09/69/' 1015 'cf252f211dbbf58bbbe01a3931092d8a8df8d55f5fe23ac5cef145aa6468/%s' % 1016 (url_filename),) 1017 hex_sum = 'b33594a2c627d72007bfa8c6d7619af699e26085' 1018 _build_and_install = ExternalPackage._build_and_install_from_package 1019 _build_and_install_current_dir = ( 1020 ExternalPackage._build_and_install_current_dir_setup_py) 1021 1022 1023class Pytz(ExternalPackage): 1024 """Pytz package.""" 1025 version = '2016.10' 1026 url_filename = 'pytz-%s.tar.gz' % version 1027 local_filename = url_filename 1028 #md5=cc9f16ba436efabdcef3c4d32ae4919c 1029 urls = ('https://pypi.python.org/packages/42/00/' 1030 '5c89fc6c9b305df84def61863528e899e9dccb196f8438f6cbe960758fc5/%s' % 1031 (url_filename),) 1032 hex_sum = '8d63f1e9b1ee862841b990a7d8ad1d4508d9f0be' 1033 _build_and_install = ExternalPackage._build_and_install_from_package 1034 _build_and_install_current_dir = ( 1035 ExternalPackage._build_and_install_current_dir_setup_py) 1036 1037 1038class Tzlocal(ExternalPackage): 1039 """Tzlocal package.""" 1040 version = '1.3' 1041 url_filename = 'tzlocal-%s.tar.gz' % version 1042 local_filename = url_filename 1043 # md5=3cb544b3975b59f91a793850a072d4a8 1044 urls = ('https://pypi.python.org/packages/d3/64/' 1045 'e4b18738496213f82b88b31c431a0e4ece143801fb6771dddd1c2bf0101b/%s' % 1046 (url_filename),) 1047 hex_sum = '730e9d7112335865a1dcfabec69c8c3086be424f' 1048 _build_and_install = ExternalPackage._build_and_install_from_package 1049 _build_and_install_current_dir = ( 1050 ExternalPackage._build_and_install_current_dir_setup_py) 1051 1052 1053class _ExternalGitRepo(ExternalPackage): 1054 """ 1055 Parent class for any package which needs to pull a git repo. 1056 1057 This class inherits from ExternalPackage only so we can sync git 1058 repos through the build_externals script. We do not reuse any of 1059 ExternalPackage's other methods. Any package that needs a git repo 1060 should subclass this and override build_and_install or fetch as 1061 they see appropriate. 1062 """ 1063 1064 os_requirements = {('/usr/bin/git') : 'git-core'} 1065 1066 # All the chromiumos projects used on the lab servers should have a 'prod' 1067 # branch used to track the software version deployed in prod. 1068 PROD_BRANCH = 'prod' 1069 MASTER_BRANCH = 'master' 1070 1071 def is_needed(self, unused_install_dir): 1072 """Tell build_externals that we need to fetch.""" 1073 # TODO(beeps): check if we're already upto date. 1074 return True 1075 1076 1077 def build_and_install(self, unused_install_dir): 1078 """ 1079 Fall through method to install a package. 1080 1081 Overwritten in base classes to pull a git repo. 1082 """ 1083 raise NotImplementedError 1084 1085 1086 def fetch(self, unused_dest_dir): 1087 """Fallthrough method to fetch a package.""" 1088 return True 1089 1090 1091class HdctoolsRepo(_ExternalGitRepo): 1092 """Clones or updates the hdctools repo.""" 1093 1094 module_name = 'servo' 1095 temp_hdctools_dir = tempfile.mktemp(suffix='hdctools') 1096 _GIT_URL = ('https://chromium.googlesource.com/' 1097 'chromiumos/third_party/hdctools') 1098 1099 def fetch(self, unused_dest_dir): 1100 """ 1101 Fetch repo to a temporary location. 1102 1103 We use an intermediate temp directory to stage our 1104 installation because we only care about the servo package. 1105 If we can't get at the top commit hash after fetching 1106 something is wrong. This can happen when we've cloned/pulled 1107 an empty repo. Not something we expect to do. 1108 1109 @parma unused_dest_dir: passed in because we inherit from 1110 ExternalPackage. 1111 1112 @return: True if repo sync was successful. 1113 """ 1114 git_repo = revision_control.GitRepo( 1115 self.temp_hdctools_dir, 1116 self._GIT_URL, 1117 None, 1118 abs_work_tree=self.temp_hdctools_dir) 1119 git_repo.reinit_repo_at(self.PROD_BRANCH) 1120 1121 if git_repo.get_latest_commit_hash(): 1122 return True 1123 return False 1124 1125 1126 def build_and_install(self, install_dir): 1127 """Reach into the hdctools repo and rsync only the servo directory.""" 1128 1129 servo_dir = os.path.join(self.temp_hdctools_dir, 'servo') 1130 if not os.path.exists(servo_dir): 1131 return False 1132 1133 rv = self._rsync(servo_dir, os.path.join(install_dir, 'servo')) 1134 shutil.rmtree(self.temp_hdctools_dir) 1135 return rv 1136 1137 1138class ChromiteRepo(_ExternalGitRepo): 1139 """Clones or updates the chromite repo.""" 1140 1141 _GIT_URL = ('https://chromium.googlesource.com/chromiumos/chromite') 1142 1143 def build_and_install(self, install_dir, master_branch=False): 1144 """ 1145 Clone if the repo isn't initialized, pull clean bits if it is. 1146 1147 Unlike it's hdctools counterpart the chromite repo clones master 1148 directly into site-packages. It doesn't use an intermediate temp 1149 directory because it doesn't need installation. 1150 1151 @param install_dir: destination directory for chromite installation. 1152 @param master_branch: if True, install master branch. Otherwise, 1153 install prod branch. 1154 """ 1155 init_branch = (self.MASTER_BRANCH if master_branch 1156 else self.PROD_BRANCH) 1157 local_chromite_dir = os.path.join(install_dir, 'chromite') 1158 git_repo = revision_control.GitRepo( 1159 local_chromite_dir, 1160 self._GIT_URL, 1161 abs_work_tree=local_chromite_dir) 1162 git_repo.reinit_repo_at(init_branch) 1163 1164 1165 if git_repo.get_latest_commit_hash(): 1166 return True 1167 return False 1168 1169 1170class DevServerRepo(_ExternalGitRepo): 1171 """Clones or updates the chromite repo.""" 1172 1173 _GIT_URL = ('https://chromium.googlesource.com/' 1174 'chromiumos/platform/dev-util') 1175 1176 def build_and_install(self, install_dir): 1177 """ 1178 Clone if the repo isn't initialized, pull clean bits if it is. 1179 1180 Unlike it's hdctools counterpart the dev-util repo clones master 1181 directly into site-packages. It doesn't use an intermediate temp 1182 directory because it doesn't need installation. 1183 1184 @param install_dir: destination directory for chromite installation. 1185 """ 1186 local_devserver_dir = os.path.join(install_dir, 'devserver') 1187 git_repo = revision_control.GitRepo(local_devserver_dir, self._GIT_URL, 1188 abs_work_tree=local_devserver_dir) 1189 git_repo.reinit_repo_at(self.PROD_BRANCH) 1190 1191 if git_repo.get_latest_commit_hash(): 1192 return True 1193 return False 1194 1195 1196class BtsocketRepo(_ExternalGitRepo): 1197 """Clones or updates the btsocket repo.""" 1198 1199 _GIT_URL = ('https://chromium.googlesource.com/' 1200 'chromiumos/platform/btsocket') 1201 1202 def fetch(self, unused_dest_dir): 1203 """ 1204 Fetch repo to a temporary location. 1205 1206 We use an intermediate temp directory because we have to build an 1207 egg for installation. If we can't get at the top commit hash after 1208 fetching something is wrong. This can happen when we've cloned/pulled 1209 an empty repo. Not something we expect to do. 1210 1211 @parma unused_dest_dir: passed in because we inherit from 1212 ExternalPackage. 1213 1214 @return: True if repo sync was successful. 1215 """ 1216 self.temp_btsocket_dir = autotemp.tempdir(unique_id='btsocket') 1217 try: 1218 git_repo = revision_control.GitRepo( 1219 self.temp_btsocket_dir.name, 1220 self._GIT_URL, 1221 None, 1222 abs_work_tree=self.temp_btsocket_dir.name) 1223 git_repo.reinit_repo_at(self.PROD_BRANCH) 1224 1225 if git_repo.get_latest_commit_hash(): 1226 return True 1227 except: 1228 self.temp_btsocket_dir.clean() 1229 raise 1230 1231 self.temp_btsocket_dir.clean() 1232 return False 1233 1234 1235 def build_and_install(self, install_dir): 1236 """ 1237 Install the btsocket module using setup.py 1238 1239 @param install_dir: Target installation directory. 1240 1241 @return: A boolean indicating success of failure. 1242 """ 1243 work_dir = os.getcwd() 1244 try: 1245 os.chdir(self.temp_btsocket_dir.name) 1246 rv = self._build_and_install_current_dir_setup_py(install_dir) 1247 finally: 1248 os.chdir(work_dir) 1249 self.temp_btsocket_dir.clean() 1250 return rv 1251