• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2#
3# Please keep this code python 2.4 compatible and stand alone.
4
5"""
6Fetch, build and install external Python library dependancies.
7
8This fetches external python libraries, builds them using your host's
9python and installs them under our own autotest/site-packages/ directory.
10
11Usage?  Just run it.
12    utils/build_externals.py
13"""
14
15import argparse
16import compileall
17import logging
18import os
19import sys
20
21import common
22from autotest_lib.client.common_lib import logging_config, logging_manager
23from autotest_lib.client.common_lib import utils
24from autotest_lib.utils import external_packages
25
26# bring in site packages as well
27utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
28
29# Where package source be fetched to relative to the top of the autotest tree.
30PACKAGE_DIR = 'ExternalSource'
31
32# Where packages will be installed to relative to the top of the autotest tree.
33INSTALL_DIR = 'site-packages'
34
35# Installs all packages, even if the system already has the version required
36INSTALL_ALL = False
37
38
39# Want to add more packages to fetch, build and install?  See the class
40# definitions at the end of external_packages.py for examples of how to do it.
41
42
43class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
44    def configure_logging(self, results_dir=None, verbose=False):
45        super(BuildExternalsLoggingConfig, self).configure_logging(
46                                                               use_console=True,
47                                                               verbose=verbose)
48
49
50def main():
51    """
52    Find all ExternalPackage classes defined in this file and ask them to
53    fetch, build and install themselves.
54    """
55    options = parse_arguments(sys.argv[1:])
56    logging_manager.configure_logging(BuildExternalsLoggingConfig(),
57                                      verbose=True)
58    os.umask(022)
59
60    top_of_tree = external_packages.find_top_of_autotest_tree()
61    package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
62    install_dir = os.path.join(top_of_tree, INSTALL_DIR)
63
64    # Make sure the install_dir is in our python module search path
65    # as well as the PYTHONPATH being used by all our setup.py
66    # install subprocesses.
67    if install_dir not in sys.path:
68        sys.path.insert(0, install_dir)
69    env_python_path_varname = 'PYTHONPATH'
70    env_python_path = os.environ.get(env_python_path_varname, '')
71    if install_dir+':' not in env_python_path:
72        os.environ[env_python_path_varname] = ':'.join([
73            install_dir, env_python_path])
74
75    fetched_packages, fetch_errors = fetch_necessary_packages(
76        package_dir, install_dir, set(options.names_to_check))
77    install_errors = build_and_install_packages(
78        fetched_packages, install_dir, options.use_chromite_master)
79
80    # Byte compile the code after it has been installed in its final
81    # location as .pyc files contain the path passed to compile_dir().
82    # When printing exception tracebacks, python uses that path first to look
83    # for the source code before checking the directory of the .pyc file.
84    # Don't leave references to our temporary build dir in the files.
85    logging.info('compiling .py files in %s to .pyc', install_dir)
86    compileall.compile_dir(install_dir, quiet=True)
87
88    # Some things install with whacky permissions, fix that.
89    external_packages.system("chmod -R a+rX '%s'" % install_dir)
90
91    errors = fetch_errors + install_errors
92    for error_msg in errors:
93        logging.error(error_msg)
94
95    return len(errors)
96
97
98def parse_arguments(args):
99    """Parse command line arguments.
100
101    @param args: The command line arguments to parse. (ususally sys.argsv[1:])
102
103    @returns An argparse.Namespace populated with argument values.
104    """
105    parser = argparse.ArgumentParser(
106            description='Command to build third party dependencies required '
107                        'for autotest.')
108    parser.add_argument('--use_chromite_master', action='store_true',
109                        help='Update chromite to master branch, rather than '
110                             'prod.')
111    parser.add_argument('--names_to_check', nargs='*', type=str, default=set(),
112                        help='Package names to check whether they are needed '
113                             'in current system.')
114    return parser.parse_args(args)
115
116
117def fetch_necessary_packages(dest_dir, install_dir, names_to_check=set()):
118    """
119    Fetches all ExternalPackages into dest_dir.
120
121    @param dest_dir: Directory the packages should be fetched into.
122    @param install_dir: Directory where packages will later installed.
123    @param names_to_check: A set of package names to check whether they are
124                           needed on current system. Default is empty.
125
126    @returns A tuple containing two lists:
127             * A list of ExternalPackage instances that were fetched and
128               need to be installed.
129             * A list of error messages for any failed fetches.
130    """
131    errors = []
132    fetched_packages = []
133    for package_class in external_packages.ExternalPackage.subclasses:
134        package = package_class()
135        if names_to_check and package.name.lower() not in names_to_check:
136            continue
137        if not package.is_needed(install_dir):
138            logging.info('A new %s is not needed on this system.',
139                         package.name)
140            if INSTALL_ALL:
141                logging.info('Installing anyways...')
142            else:
143                continue
144        if not package.fetch(dest_dir):
145            msg = 'Unable to download %s' % package.name
146            logging.error(msg)
147            errors.append(msg)
148        else:
149            fetched_packages.append(package)
150
151    return fetched_packages, errors
152
153
154def build_and_install_packages(packages, install_dir,
155                               use_chromite_master=False):
156    """
157    Builds and installs all packages into install_dir.
158
159    @param packages - A list of already fetched ExternalPackage instances.
160    @param install_dir - Directory the packages should be installed into.
161    @param use_chromite_master: True if updating chromite to master branch.
162
163    @returns A list of error messages for any installs that failed.
164    """
165    errors = []
166    for package in packages:
167        if use_chromite_master and package.name.lower() == 'chromiterepo':
168            result = package.build_and_install(install_dir, master_branch=True)
169        else:
170            result = package.build_and_install(install_dir)
171        if isinstance(result, bool):
172            success = result
173            message = None
174        else:
175            success = result[0]
176            message = result[1]
177        if not success:
178            msg = ('Unable to build and install %s.\nError: %s' %
179                   (package.name, message))
180            logging.error(msg)
181            errors.append(msg)
182    return errors
183
184
185if __name__ == '__main__':
186    sys.exit(main())
187