1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import tempfile 8 9import common 10from autotest_lib.client.bin import utils as common_utils 11from autotest_lib.client.common_lib import error 12from autotest_lib.client.common_lib.cros import dev_server 13from autotest_lib.client.common_lib.cros import retry 14from autotest_lib.server import utils as server_utils 15from autotest_lib.site_utils.lxc import constants 16 17try: 18 from chromite.lib import metrics 19except ImportError: 20 metrics = common_utils.metrics_mock 21 22 23def get_container_info(container_path, **filters): 24 """Get a collection of container information in the given container path. 25 26 This method parse the output of lxc-ls to get a list of container 27 information. The lxc-ls command output looks like: 28 NAME STATE IPV4 IPV6 AUTOSTART PID MEMORY RAM SWAP 29 -------------------------------------------------------------------------- 30 base STOPPED - - NO - - - - 31 test_123 RUNNING 10.0.3.27 - NO 8359 6.28MB 6.28MB 0.0MB 32 33 @param container_path: Path to look for containers. 34 @param filters: Key value to filter the containers, e.g., name='base' 35 36 @return: A list of dictionaries that each dictionary has the information of 37 a container. The keys are defined in ATTRIBUTES. 38 """ 39 cmd = 'sudo lxc-ls -P %s -f -F %s' % (os.path.realpath(container_path), 40 ','.join(constants.ATTRIBUTES)) 41 output = common_utils.run(cmd).stdout 42 info_collection = [] 43 44 logging.info('cmd [%s] output:\n%s', cmd, output) 45 46 for line in output.splitlines()[1:]: 47 # Only LXC 1.x has the second line of '-' as a separator. 48 if line.startswith('------'): 49 continue 50 info_collection.append(dict(zip(constants.ATTRIBUTES, line.split()))) 51 if filters: 52 filtered_collection = [] 53 for key, value in filters.iteritems(): 54 for info in info_collection: 55 if key in info and info[key] == value: 56 filtered_collection.append(info) 57 info_collection = filtered_collection 58 return info_collection 59 60 61def download_extract(url, target, extract_dir): 62 """Download the file from given url and save it to the target, then extract. 63 64 @param url: Url to download the file. 65 @param target: Path of the file to save to. 66 @param extract_dir: Directory to extract the content of the file to. 67 """ 68 remote_url = dev_server.DevServer.get_server_url(url) 69 # This can be run in multiple threads, pick a unique tmp_file.name. 70 with tempfile.NamedTemporaryFile(prefix=os.path.basename(target) + '_', 71 delete=False) as tmp_file: 72 if remote_url in dev_server.ImageServerBase.servers(): 73 _download_via_devserver(url, tmp_file.name) 74 else: 75 _download_via_curl(url, tmp_file.name) 76 common_utils.run('sudo mv %s %s' % (tmp_file.name, target)) 77 common_utils.run('sudo tar -xvf %s -C %s' % (target, extract_dir)) 78 79 80# Make sure retries only happen in the non-timeout case. 81@retry.retry((error.CmdError), 82 raiselist=[error.CmdTimeoutError], 83 timeout_min=3*2, 84 delay_sec=10) 85def _download_via_curl(url, target_file_path): 86 # We do not want to retry on CmdTimeoutError but still retry on 87 # CmdError. Hence we can't use curl --timeout=... 88 common_utils.run('sudo curl -s %s -o %s' % (url, target_file_path), 89 stderr_tee=common_utils.TEE_TO_LOGS, timeout=3*60) 90 91 92# Make sure retries only happen in the non-timeout case. 93@retry.retry((error.CmdError), 94 raiselist=[error.CmdTimeoutError], 95 timeout_min=(constants.DEVSERVER_CALL_TIMEOUT * 96 constants.DEVSERVER_CALL_RETRY / 60), 97 delay_sec=constants.DEVSERVER_CALL_DELAY) 98def _download_via_devserver(url, target_file_path): 99 dev_server.ImageServerBase.download_file( 100 url, target_file_path, timeout=constants.DEVSERVER_CALL_TIMEOUT) 101 102 103def _install_package_precheck(packages): 104 """If SSP is not enabled or the test is running in chroot (using test_that), 105 packages installation should be skipped. 106 107 The check does not raise exception so tests started by test_that or running 108 in an Autotest setup with SSP disabled can continue. That assume the running 109 environment, chroot or a machine, has the desired packages installed 110 already. 111 112 @param packages: A list of names of the packages to install. 113 114 @return: True if package installation can continue. False if it should be 115 skipped. 116 117 """ 118 if server_utils.is_inside_chroot(): 119 logging.info('Test is running inside chroot. Install package %s is ' 120 'skipped.', packages) 121 return False 122 123 if not common_utils.is_in_container(): 124 raise error.ContainerError('Package installation is only supported ' 125 'when test is running inside container.') 126 127 return True 128 129 130def _remove_banned_packages(packages, banned_packages): 131 """Filter out packages. 132 133 @param packages: A set of packages names that have been requested. 134 @param items: A list of package names that are not to be installed. 135 136 @return: A sanatized set of packages names to install. 137 """ 138 return {package for package in packages if package not in banned_packages} 139 140 141def _ensure_pip(target_setting): 142 """ Ensure pip is installed, if not install it. 143 144 @param target_setting: target command param specifying the path to where 145 python packages should be installed. 146 """ 147 try: 148 import pip 149 except ImportError: 150 common_utils.run( 151 'wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py') 152 common_utils.run('python /tmp/get-pip.py %s' % target_setting) 153 154 155@metrics.SecondsTimerDecorator( 156 '%s/install_packages_duration' % constants.STATS_KEY) 157@retry.retry(error.CmdError, timeout_min=30) 158def install_packages(packages=[], python_packages=[], force_latest=False): 159 """Install the given package inside container. 160 161 !!! WARNING !!! 162 This call may introduce several minutes of delay in test run. The best way 163 to avoid such delay is to update the base container used for the test run. 164 File a bug for infra deputy to update the base container with the new 165 package a test requires. 166 167 @param packages: A list of names of the packages to install. 168 @param python_packages: A list of names of the python packages to install 169 using pip. 170 @param force_latest: True to force to install the latest version of the 171 package. Default to False, which means skip installing 172 the package if it's installed already, even with an old 173 version. 174 175 @raise error.ContainerError: If package is attempted to be installed outside 176 a container. 177 @raise error.CmdError: If the package doesn't exist or failed to install. 178 179 """ 180 if not _install_package_precheck(packages or python_packages): 181 return 182 183 # If force_latest is False, only install packages that are not already 184 # installed. 185 if not force_latest: 186 packages = [p for p in packages 187 if not common_utils.is_package_installed(p)] 188 python_packages = [p for p in python_packages 189 if not common_utils.is_python_package_installed(p)] 190 if not packages and not python_packages: 191 logging.debug( 192 'All packages are installed already, skip reinstall.') 193 return 194 195 # Always run apt-get update before installing any container. The base 196 # container may have outdated cache. 197 common_utils.run('sudo apt-get update') 198 199 # Make sure the lists are not None for iteration. 200 packages = [] if not packages else packages 201 # Remove duplicates. 202 packages = set(packages) 203 204 # Ubuntu distribution of pip is very old, do not use it as it causes 205 # segmentation faults. Some tests request these packages, ensure they 206 # are not installed. 207 packages = _remove_banned_packages(packages, ['python-pip', 'python-dev']) 208 209 if packages: 210 common_utils.run( 211 'sudo DEBIAN_FRONTEND=noninteractive apt-get install %s -y ' 212 '--force-yes' % ' '.join(packages)) 213 logging.debug('Packages are installed: %s.', packages) 214 215 target_setting = '' 216 # For containers running in Moblab, /usr/local/lib/python2.7/dist-packages/ 217 # is a readonly mount from the host. Therefore, new python modules have to 218 # be installed in /usr/lib/python2.7/dist-packages/ 219 # Containers created in Moblab does not have autotest/site-packages folder. 220 if not os.path.exists('/usr/local/autotest/site-packages'): 221 target_setting = '--target="/usr/lib/python2.7/dist-packages/"' 222 # Pip should be installed in the base container, if not install it. 223 if python_packages: 224 _ensure_pip(target_setting) 225 common_utils.run('python -m pip install pip --upgrade') 226 common_utils.run('python -m pip install %s %s' % (target_setting, 227 ' '.join(python_packages))) 228 logging.debug('Python packages are installed: %s.', python_packages) 229