• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2#
3# Please keep this code python 2.4 compatible and standalone.
4
5"""
6Fetch, build and install external Python library dependancies.
7
8This fetches external python libraries, builds them using your host's
9python and installs them under our own autotest/site-packages/ directory.
10
11Usage?  Just run it.
12    utils/build_externals.py
13"""
14
15import argparse
16import compileall
17import logging
18import os
19import sys
20
21import common
22from autotest_lib.client.common_lib import logging_config, logging_manager
23from autotest_lib.client.common_lib import utils
24from autotest_lib.utils import external_packages
25
26# bring in site packages as well
27utils.import_site_module(__file__, 'autotest_lib.utils.site_external_packages')
28
29# Where package source be fetched to relative to the top of the autotest tree.
30PACKAGE_DIR = 'ExternalSource'
31
32# Where packages will be installed to relative to the top of the autotest tree.
33INSTALL_DIR = 'site-packages'
34
35# Installs all packages, even if the system already has the version required
36INSTALL_ALL = False
37
38
39# Want to add more packages to fetch, build and install?  See the class
40# definitions at the end of external_packages.py for examples of how to do it.
41
42
43class BuildExternalsLoggingConfig(logging_config.LoggingConfig):
44    """Logging manager config."""
45
46    def configure_logging(self, results_dir=None, verbose=False):
47        """Configure logging."""
48        super(BuildExternalsLoggingConfig, self).configure_logging(
49                                                               use_console=True,
50                                                               verbose=verbose)
51
52
53def main():
54    """
55    Find all ExternalPackage classes defined in this file and ask them to
56    fetch, build and install themselves.
57    """
58    options = parse_arguments(sys.argv[1:])
59    logging_manager.configure_logging(BuildExternalsLoggingConfig(),
60                                      verbose=True)
61    os.umask(022)
62
63    top_of_tree = external_packages.find_top_of_autotest_tree()
64    package_dir = os.path.join(top_of_tree, PACKAGE_DIR)
65    install_dir = os.path.join(top_of_tree, INSTALL_DIR)
66
67    # Make sure the install_dir is in our python module search path
68    # as well as the PYTHONPATH being used by all our setup.py
69    # install subprocesses.
70    if install_dir not in sys.path:
71        sys.path.insert(0, install_dir)
72    env_python_path_varname = 'PYTHONPATH'
73    env_python_path = os.environ.get(env_python_path_varname, '')
74    if install_dir+':' not in env_python_path:
75        os.environ[env_python_path_varname] = ':'.join([
76            install_dir, env_python_path])
77
78    fetched_packages, fetch_errors = fetch_necessary_packages(
79        package_dir, install_dir, set(options.names_to_check))
80    install_errors = build_and_install_packages(
81        fetched_packages, install_dir, options.use_chromite_master)
82
83    # Byte compile the code after it has been installed in its final
84    # location as .pyc files contain the path passed to compile_dir().
85    # When printing exception tracebacks, python uses that path first to look
86    # for the source code before checking the directory of the .pyc file.
87    # Don't leave references to our temporary build dir in the files.
88    logging.info('compiling .py files in %s to .pyc', install_dir)
89    compileall.compile_dir(install_dir, quiet=True)
90
91    # Some things install with whacky permissions, fix that.
92    external_packages.system("chmod -R a+rX '%s'" % install_dir)
93
94    errors = fetch_errors + install_errors
95    for error_msg in errors:
96        logging.error(error_msg)
97
98    if not errors:
99      logging.info("Syntax errors from pylint above are expected, not "
100                   "problematic. SUCCESS.")
101    else:
102      logging.info("Problematic errors encountered. FAILURE.")
103    return len(errors)
104
105
106def parse_arguments(args):
107    """Parse command line arguments.
108
109    @param args: The command line arguments to parse. (ususally sys.argsv[1:])
110
111    @returns An argparse.Namespace populated with argument values.
112    """
113    parser = argparse.ArgumentParser(
114            description='Command to build third party dependencies required '
115                        'for autotest.')
116    parser.add_argument('--use_chromite_master', action='store_true',
117                        help='Update chromite to master branch, rather than '
118                             'prod.')
119    parser.add_argument('--names_to_check', nargs='*', type=str, default=set(),
120                        help='Package names to check whether they are needed '
121                             'in current system.')
122    return parser.parse_args(args)
123
124
125def fetch_necessary_packages(dest_dir, install_dir, names_to_check=set()):
126    """
127    Fetches all ExternalPackages into dest_dir.
128
129    @param dest_dir: Directory the packages should be fetched into.
130    @param install_dir: Directory where packages will later installed.
131    @param names_to_check: A set of package names to check whether they are
132                           needed on current system. Default is empty.
133
134    @returns A tuple containing two lists:
135             * A list of ExternalPackage instances that were fetched and
136               need to be installed.
137             * A list of error messages for any failed fetches.
138    """
139    errors = []
140    fetched_packages = []
141    for package_class in external_packages.ExternalPackage.subclasses:
142        package = package_class()
143        if names_to_check and package.name.lower() not in names_to_check:
144            continue
145        if not package.is_needed(install_dir):
146            logging.info('A new %s is not needed on this system.',
147                         package.name)
148            if INSTALL_ALL:
149                logging.info('Installing anyways...')
150            else:
151                continue
152        if not package.fetch(dest_dir):
153            msg = 'Unable to download %s' % package.name
154            logging.error(msg)
155            errors.append(msg)
156        else:
157            fetched_packages.append(package)
158
159    return fetched_packages, errors
160
161
162def build_and_install_packages(packages, install_dir,
163                               use_chromite_master=False):
164    """
165    Builds and installs all packages into install_dir.
166
167    @param packages - A list of already fetched ExternalPackage instances.
168    @param install_dir - Directory the packages should be installed into.
169    @param use_chromite_master: True if updating chromite to master branch.
170
171    @returns A list of error messages for any installs that failed.
172    """
173    errors = []
174    for package in packages:
175        if use_chromite_master and package.name.lower() == 'chromiterepo':
176            result = package.build_and_install(install_dir, master_branch=True)
177        else:
178            result = package.build_and_install(install_dir)
179        if isinstance(result, bool):
180            success = result
181            message = None
182        else:
183            success = result[0]
184            message = result[1]
185        if not success:
186            msg = ('Unable to build and install %s.\nError: %s' %
187                   (package.name, message))
188            logging.error(msg)
189            errors.append(msg)
190    return errors
191
192
193if __name__ == '__main__':
194    sys.exit(main())
195