1# Copyright 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import time 8 9import common 10 11from autotest_lib.client.bin import utils 12from autotest_lib.client.common_lib import error 13from autotest_lib.site_utils.lxc import config as lxc_config 14from autotest_lib.site_utils.lxc import constants 15from autotest_lib.site_utils.lxc import lxc 16from autotest_lib.site_utils.lxc import utils as lxc_utils 17from autotest_lib.site_utils.lxc.cleanup_if_fail import cleanup_if_fail 18from autotest_lib.site_utils.lxc.base_image import BaseImage 19from autotest_lib.site_utils.lxc.constants import \ 20 CONTAINER_POOL_METRICS_PREFIX as METRICS_PREFIX 21from autotest_lib.site_utils.lxc.container import Container 22from autotest_lib.site_utils.lxc.container_factory import ContainerFactory 23 24try: 25 from chromite.lib import metrics 26 from infra_libs import ts_mon 27except ImportError: 28 import mock 29 metrics = utils.metrics_mock 30 ts_mon = mock.Mock() 31 32 33class ContainerBucket(object): 34 """A wrapper class to interact with containers in a specific container path. 35 """ 36 37 def __init__(self, container_path=constants.DEFAULT_CONTAINER_PATH, 38 container_factory=None): 39 """Initialize a ContainerBucket. 40 41 @param container_path: Path to the directory used to store containers. 42 Default is set to AUTOSERV/container_path in 43 global config. 44 @param container_factory: A factory for creating Containers. 45 """ 46 self.container_path = os.path.realpath(container_path) 47 if container_factory is not None: 48 self._factory = container_factory 49 else: 50 # Pass in the container path so that the bucket is hermetic (i.e. so 51 # that if the container path is customized, the base image doesn't 52 # fall back to using the default container path). 53 try: 54 base_image_ok = True 55 container = BaseImage(self.container_path).get() 56 except error.ContainerError as e: 57 base_image_ok = False 58 raise e 59 finally: 60 metrics.Counter(METRICS_PREFIX + '/base_image', 61 field_spec=[ts_mon.BooleanField('corrupted')] 62 ).increment( 63 fields={'corrupted': not base_image_ok}) 64 self._factory = ContainerFactory( 65 base_container=container, 66 lxc_path=self.container_path) 67 self.container_cache = {} 68 69 70 def get_all(self, force_update=False): 71 """Get details of all containers. 72 73 Retrieves all containers owned by the bucket. Note that this doesn't 74 include the base container, or any containers owned by the container 75 pool. 76 77 @param force_update: Boolean, ignore cached values if set. 78 79 @return: A dictionary of all containers with detailed attributes, 80 indexed by container name. 81 """ 82 logging.debug("Fetching all extant LXC containers") 83 info_collection = lxc.get_container_info(self.container_path) 84 if force_update: 85 logging.debug("Clearing cached container info") 86 containers = {} if force_update else self.container_cache 87 for info in info_collection: 88 if info["name"] in containers: 89 continue 90 container = Container.create_from_existing_dir(self.container_path, 91 **info) 92 # Active containers have an ID. Zygotes and base containers, don't. 93 if container.id is not None: 94 containers[container.id] = container 95 self.container_cache = containers 96 return containers 97 98 99 def get_container(self, container_id): 100 """Get a container with matching name. 101 102 @param container_id: ID of the container. 103 104 @return: A container object with matching name. Returns None if no 105 container matches the given name. 106 """ 107 logging.debug("Fetching LXC container with id %s", container_id) 108 if container_id in self.container_cache: 109 logging.debug("Found container %s in cache", container_id) 110 return self.container_cache[container_id] 111 112 container = self.get_all().get(container_id, None) 113 if None == container: 114 logging.debug("Could not find container %s", container_id) 115 return container 116 117 118 def exist(self, container_id): 119 """Check if a container exists with the given name. 120 121 @param container_id: ID of the container. 122 123 @return: True if the container with the given ID exists, otherwise 124 returns False. 125 """ 126 return self.get_container(container_id) != None 127 128 129 def destroy_all(self): 130 """Destroy all containers, base must be destroyed at the last. 131 """ 132 containers = self.get_all().values() 133 for container in sorted( 134 containers, key=lambda n: 1 if n.name == constants.BASE else 0): 135 key = container.id 136 logging.info('Destroy container %s.', container.name) 137 container.destroy() 138 del self.container_cache[key] 139 140 def scrub_container_location(self, name, 141 timeout=constants.LXC_SCRUB_TIMEOUT): 142 """Destroy a possibly-nonexistent, possibly-malformed container. 143 144 This exists to clean up an unreachable container which may or may not 145 exist and is probably but not definitely malformed if it does exist. It 146 is accordingly scorched-earth and force-destroys the container with all 147 associated snapshots. Also accordingly, this will not raise an 148 exception if the destruction fails. 149 150 @param name: ID of the container. 151 @param timeout: Seconds to wait for removal. 152 153 @returns: CmdResult object from the shell command 154 """ 155 logging.debug( 156 "Force-destroying container %s if it exists, with timeout %s sec", 157 name, timeout) 158 try: 159 result = lxc_utils.destroy( 160 self.container_path, name, 161 force=True, snapshots=True, ignore_status=True, timeout=timeout 162 ) 163 except error.CmdTimeoutError: 164 logging.warning("Force-destruction of container %s timed out.", name) 165 logging.debug("Force-destruction exit code %s", result.exit_status) 166 return result 167 168 169 170 @metrics.SecondsTimerDecorator( 171 '%s/setup_test_duration' % constants.STATS_KEY) 172 @cleanup_if_fail() 173 def setup_test(self, container_id, job_id, server_package_url, result_path, 174 control=None, skip_cleanup=False, job_folder=None, 175 dut_name=None, isolate_hash=None): 176 """Setup test container for the test job to run. 177 178 The setup includes: 179 1. Install autotest_server package from given url. 180 2. Copy over local shadow_config.ini. 181 3. Mount local site-packages. 182 4. Mount test result directory. 183 184 TODO(dshi): Setup also needs to include test control file for autoserv 185 to run in container. 186 187 @param container_id: ID to assign to the test container. 188 @param job_id: Job id for the test job to run in the test container. 189 @param server_package_url: Url to download autotest_server package. 190 @param result_path: Directory to be mounted to container to store test 191 results. 192 @param control: Path to the control file to run the test job. Default is 193 set to None. 194 @param skip_cleanup: Set to True to skip cleanup, used to troubleshoot 195 container failures. 196 @param job_folder: Folder name of the job, e.g., 123-debug_user. 197 @param dut_name: Name of the dut to run test, used as the hostname of 198 the container. Default is None. 199 @param isolate_hash: String key to look up the isolate package needed 200 to run test. Default is None, supersedes 201 server_package_url if present. 202 @return: A Container object for the test container. 203 204 @raise ContainerError: If container does not exist, or not running. 205 """ 206 start_time = time.time() 207 208 if not os.path.exists(result_path): 209 raise error.ContainerError('Result directory does not exist: %s', 210 result_path) 211 result_path = os.path.abspath(result_path) 212 213 # Save control file to result_path temporarily. The reason is that the 214 # control file in drone_tmp folder can be deleted during scheduler 215 # restart. For test not using SSP, the window between test starts and 216 # control file being picked up by the test is very small (< 2 seconds). 217 # However, for tests using SSP, it takes around 1 minute before the 218 # container is setup. If scheduler is restarted during that period, the 219 # control file will be deleted, and the test will fail. 220 if control: 221 control_file_name = os.path.basename(control) 222 safe_control = os.path.join(result_path, control_file_name) 223 utils.run('cp %s %s' % (control, safe_control)) 224 225 # Create test container from the base container. 226 container = self._factory.create_container(container_id) 227 228 # Deploy server side package 229 if isolate_hash: 230 container.install_ssp_isolate(isolate_hash) 231 else: 232 container.install_ssp(server_package_url) 233 234 deploy_config_manager = lxc_config.DeployConfigManager(container) 235 deploy_config_manager.deploy_pre_start() 236 237 # Copy over control file to run the test job. 238 if control: 239 container.install_control_file(safe_control) 240 241 # Use a pre-packaged Trusty-compatible Autotest site_packages 242 # instead if it exists. crbug.com/1013241 243 if os.path.exists(constants.TRUSTY_SITE_PACKAGES_PATH): 244 mount_entries = [(constants.TRUSTY_SITE_PACKAGES_PATH, 245 constants.CONTAINER_SITE_PACKAGES_PATH, 246 True)] 247 else: 248 mount_entries = [(constants.SITE_PACKAGES_PATH, 249 constants.CONTAINER_SITE_PACKAGES_PATH, 250 True)] 251 mount_entries.extend([ 252 (result_path, 253 os.path.join(constants.RESULT_DIR_FMT % job_folder), 254 False), 255 ]) 256 257 # Update container config to mount directories. 258 for source, destination, readonly in mount_entries: 259 container.mount_dir(source, destination, readonly) 260 261 # Update file permissions. 262 # TODO(dshi): crbug.com/459344 Skip following action when test container 263 # can be unprivileged container. 264 autotest_path = os.path.join( 265 container.rootfs, 266 constants.CONTAINER_AUTOTEST_DIR.lstrip(os.path.sep)) 267 utils.run('sudo chown -R root "%s"' % autotest_path) 268 utils.run('sudo chgrp -R root "%s"' % autotest_path) 269 270 container.start(wait_for_network=True) 271 deploy_config_manager.deploy_post_start() 272 273 # Update the hostname of the test container to be `dut-name`. 274 # Some TradeFed tests use hostname in test results, which is used to 275 # group test results in dashboard. The default container name is set to 276 # be the name of the folder, which is unique (as it is composed of job 277 # id and timestamp. For better result view, the container's hostname is 278 # set to be a string containing the dut hostname. 279 if dut_name: 280 container.set_hostname(constants.CONTAINER_UTSNAME_FORMAT % 281 dut_name.replace('.', '-')) 282 283 container.modify_import_order() 284 285 container.verify_autotest_setup(job_folder) 286 287 logging.debug('Test container %s is set up.', container.name) 288 return container 289