1# Copyright 2017 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import time 8 9import common 10 11from autotest_lib.client.bin import utils 12from autotest_lib.client.common_lib import error 13from autotest_lib.client.common_lib.global_config import global_config 14from autotest_lib.site_utils.lxc import config as lxc_config 15from autotest_lib.site_utils.lxc import constants 16from autotest_lib.site_utils.lxc import lxc 17from autotest_lib.site_utils.lxc.cleanup_if_fail import cleanup_if_fail 18from autotest_lib.site_utils.lxc.base_image import BaseImage 19from autotest_lib.site_utils.lxc.constants import \ 20 CONTAINER_POOL_METRICS_PREFIX as METRICS_PREFIX 21from autotest_lib.site_utils.lxc.container import Container 22from autotest_lib.site_utils.lxc.container_factory import ContainerFactory 23 24try: 25 from chromite.lib import metrics 26 from infra_libs import ts_mon 27except ImportError: 28 import mock 29 metrics = utils.metrics_mock 30 ts_mon = mock.Mock() 31 32 33# Timeout (in seconds) for container pool operations. 34_CONTAINER_POOL_TIMEOUT = 3 35 36_USE_LXC_POOL = global_config.get_config_value('LXC_POOL', 'use_lxc_pool', 37 type=bool) 38 39class ContainerBucket(object): 40 """A wrapper class to interact with containers in a specific container path. 41 """ 42 43 def __init__(self, container_path=constants.DEFAULT_CONTAINER_PATH, 44 container_factory=None): 45 """Initialize a ContainerBucket. 46 47 @param container_path: Path to the directory used to store containers. 48 Default is set to AUTOSERV/container_path in 49 global config. 50 @param container_factory: A factory for creating Containers. 51 """ 52 self.container_path = os.path.realpath(container_path) 53 if container_factory is not None: 54 self._factory = container_factory 55 else: 56 # Pass in the container path so that the bucket is hermetic (i.e. so 57 # that if the container path is customized, the base image doesn't 58 # fall back to using the default container path). 59 try: 60 base_image_ok = True 61 container = BaseImage(self.container_path).get() 62 except error.ContainerError as e: 63 base_image_ok = False 64 raise e 65 finally: 66 metrics.Counter(METRICS_PREFIX + '/base_image', 67 field_spec=[ts_mon.BooleanField('corrupted')] 68 ).increment( 69 fields={'corrupted': not base_image_ok}) 70 self._factory = ContainerFactory( 71 base_container=container, 72 lxc_path=self.container_path) 73 self.container_cache = {} 74 75 76 def get_all(self, force_update=False): 77 """Get details of all containers. 78 79 Retrieves all containers owned by the bucket. Note that this doesn't 80 include the base container, or any containers owned by the container 81 pool. 82 83 @param force_update: Boolean, ignore cached values if set. 84 85 @return: A dictionary of all containers with detailed attributes, 86 indexed by container name. 87 """ 88 info_collection = lxc.get_container_info(self.container_path) 89 containers = {} if force_update else self.container_cache 90 for info in info_collection: 91 if info["name"] in containers: 92 continue 93 container = Container.create_from_existing_dir(self.container_path, 94 **info) 95 # Active containers have an ID. Zygotes and base containers, don't. 96 if container.id is not None: 97 containers[container.id] = container 98 self.container_cache = containers 99 return containers 100 101 102 def get_container(self, container_id): 103 """Get a container with matching name. 104 105 @param container_id: ID of the container. 106 107 @return: A container object with matching name. Returns None if no 108 container matches the given name. 109 """ 110 if container_id in self.container_cache: 111 return self.container_cache[container_id] 112 113 return self.get_all().get(container_id, None) 114 115 116 def exist(self, container_id): 117 """Check if a container exists with the given name. 118 119 @param container_id: ID of the container. 120 121 @return: True if the container with the given ID exists, otherwise 122 returns False. 123 """ 124 return self.get_container(container_id) != None 125 126 127 def destroy_all(self): 128 """Destroy all containers, base must be destroyed at the last. 129 """ 130 containers = self.get_all().values() 131 for container in sorted( 132 containers, key=lambda n: 1 if n.name == constants.BASE else 0): 133 key = container.id 134 logging.info('Destroy container %s.', container.name) 135 container.destroy() 136 del self.container_cache[key] 137 138 139 140 @metrics.SecondsTimerDecorator( 141 '%s/setup_test_duration' % constants.STATS_KEY) 142 @cleanup_if_fail() 143 def setup_test(self, container_id, job_id, server_package_url, result_path, 144 control=None, skip_cleanup=False, job_folder=None, 145 dut_name=None, isolate_hash=None): 146 """Setup test container for the test job to run. 147 148 The setup includes: 149 1. Install autotest_server package from given url. 150 2. Copy over local shadow_config.ini. 151 3. Mount local site-packages. 152 4. Mount test result directory. 153 154 TODO(dshi): Setup also needs to include test control file for autoserv 155 to run in container. 156 157 @param container_id: ID to assign to the test container. 158 @param job_id: Job id for the test job to run in the test container. 159 @param server_package_url: Url to download autotest_server package. 160 @param result_path: Directory to be mounted to container to store test 161 results. 162 @param control: Path to the control file to run the test job. Default is 163 set to None. 164 @param skip_cleanup: Set to True to skip cleanup, used to troubleshoot 165 container failures. 166 @param job_folder: Folder name of the job, e.g., 123-debug_user. 167 @param dut_name: Name of the dut to run test, used as the hostname of 168 the container. Default is None. 169 @param isolate_hash: String key to look up the isolate package needed 170 to run test. Default is None, supersedes 171 server_package_url if present. 172 @return: A Container object for the test container. 173 174 @raise ContainerError: If container does not exist, or not running. 175 """ 176 start_time = time.time() 177 178 if not os.path.exists(result_path): 179 raise error.ContainerError('Result directory does not exist: %s', 180 result_path) 181 result_path = os.path.abspath(result_path) 182 183 # Save control file to result_path temporarily. The reason is that the 184 # control file in drone_tmp folder can be deleted during scheduler 185 # restart. For test not using SSP, the window between test starts and 186 # control file being picked up by the test is very small (< 2 seconds). 187 # However, for tests using SSP, it takes around 1 minute before the 188 # container is setup. If scheduler is restarted during that period, the 189 # control file will be deleted, and the test will fail. 190 if control: 191 control_file_name = os.path.basename(control) 192 safe_control = os.path.join(result_path, control_file_name) 193 utils.run('cp %s %s' % (control, safe_control)) 194 195 # Create test container from the base container. 196 container = self._factory.create_container(container_id) 197 198 # Deploy server side package 199 if isolate_hash: 200 container.install_ssp_isolate(isolate_hash) 201 else: 202 container.install_ssp(server_package_url) 203 204 deploy_config_manager = lxc_config.DeployConfigManager(container) 205 deploy_config_manager.deploy_pre_start() 206 207 # Copy over control file to run the test job. 208 if control: 209 container.install_control_file(safe_control) 210 211 mount_entries = [(constants.SITE_PACKAGES_PATH, 212 constants.CONTAINER_SITE_PACKAGES_PATH, 213 True), 214 (result_path, 215 os.path.join(constants.RESULT_DIR_FMT % job_folder), 216 False), 217 ] 218 219 # Update container config to mount directories. 220 for source, destination, readonly in mount_entries: 221 container.mount_dir(source, destination, readonly) 222 223 # Update file permissions. 224 # TODO(dshi): crbug.com/459344 Skip following action when test container 225 # can be unprivileged container. 226 autotest_path = os.path.join( 227 container.rootfs, 228 constants.CONTAINER_AUTOTEST_DIR.lstrip(os.path.sep)) 229 utils.run('sudo chown -R root "%s"' % autotest_path) 230 utils.run('sudo chgrp -R root "%s"' % autotest_path) 231 232 container.start(wait_for_network=True) 233 deploy_config_manager.deploy_post_start() 234 235 # Update the hostname of the test container to be `dut-name`. 236 # Some TradeFed tests use hostname in test results, which is used to 237 # group test results in dashboard. The default container name is set to 238 # be the name of the folder, which is unique (as it is composed of job 239 # id and timestamp. For better result view, the container's hostname is 240 # set to be a string containing the dut hostname. 241 if dut_name: 242 container.set_hostname(constants.CONTAINER_UTSNAME_FORMAT % 243 dut_name.replace('.', '-')) 244 245 container.modify_import_order() 246 247 container.verify_autotest_setup(job_folder) 248 249 logging.debug('Test container %s is set up.', container.name) 250 return container 251