1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import collections 6import json 7import logging 8import os 9import re 10import tempfile 11import time 12 13import common 14from autotest_lib.client.bin import utils 15from autotest_lib.client.common_lib import error 16from autotest_lib.site_utils.lxc import constants 17from autotest_lib.site_utils.lxc import lxc 18from autotest_lib.site_utils.lxc import utils as lxc_utils 19 20try: 21 from chromite.lib import metrics 22except ImportError: 23 metrics = utils.metrics_mock 24 25from chromite.lib import constants as chromite_constants 26 27# Naming convention of test container, e.g., test_300_1422862512_2424, where: 28# 300: The test job ID. 29# 1422862512: The tick when container is created. 30# 2424: The PID of autoserv that starts the container. 31_TEST_CONTAINER_NAME_FMT = 'test_%s_%d_%d' 32# Name of the container ID file. 33_CONTAINER_ID_FILENAME = 'container_id.json' 34 35 36class ContainerId(collections.namedtuple('ContainerId', 37 ['job_id', 'creation_time', 'pid'])): 38 """An identifier for containers.""" 39 40 # Optimization. Avoids __dict__ creation. Empty because this subclass has 41 # no instance vars of its own. 42 __slots__ = () 43 44 45 def __str__(self): 46 return _TEST_CONTAINER_NAME_FMT % self 47 48 49 def save(self, path): 50 """Saves the ID to the given path. 51 52 @param path: Path to a directory where the container ID will be 53 serialized. 54 """ 55 dst = os.path.join(path, _CONTAINER_ID_FILENAME) 56 with open(dst, 'w') as f: 57 json.dump(self, f) 58 59 @classmethod 60 def load(cls, path): 61 """Reads the ID from the given path. 62 63 @param path: Path to check for a serialized container ID. 64 65 @return: A container ID if one is found on the given path, or None 66 otherwise. 67 68 @raise ValueError: If a JSON load error occurred. 69 @raise TypeError: If the file was valid JSON but didn't contain a valid 70 ContainerId. 71 """ 72 src = os.path.join(path, _CONTAINER_ID_FILENAME) 73 74 try: 75 with open(src, 'r') as f: 76 job_id, ctime, pid = json.load(f) 77 except IOError: 78 # File not found, or couldn't be opened for some other reason. 79 # Treat all these cases as no ID. 80 return None 81 # TODO(pprabhu, crbug.com/842343) Remove this once all persistent 82 # container ids have migrated to str. 83 job_id = str(job_id) 84 return cls(job_id, ctime, pid) 85 86 87 @classmethod 88 def create(cls, job_id, ctime=None, pid=None): 89 """Creates a new container ID. 90 91 @param job_id: The first field in the ID. 92 @param ctime: The second field in the ID. Optional. If not provided, 93 the current epoch timestamp is used. 94 @param pid: The third field in the ID. Optional. If not provided, the 95 PID of the current process is used. 96 """ 97 if ctime is None: 98 ctime = int(time.time()) 99 if pid is None: 100 pid = os.getpid() 101 # TODO(pprabhu) Drop str() cast once 102 # job_directories.get_job_id_or_task_id() starts returning str directly. 103 return cls(str(job_id), ctime, pid) 104 105 106class Container(object): 107 """A wrapper class of an LXC container. 108 109 The wrapper class provides methods to interact with a container, e.g., 110 start, stop, destroy, run a command. It also has attributes of the 111 container, including: 112 name: Name of the container. 113 state: State of the container, e.g., ABORTING, RUNNING, STARTING, STOPPED, 114 or STOPPING. 115 116 lxc-ls can also collect other attributes of a container including: 117 ipv4: IP address for IPv4. 118 ipv6: IP address for IPv6. 119 autostart: If the container will autostart at system boot. 120 pid: Process ID of the container. 121 memory: Memory used by the container, as a string, e.g., "6.2MB" 122 ram: Physical ram used by the container, as a string, e.g., "6.2MB" 123 swap: swap used by the container, as a string, e.g., "1.0MB" 124 125 For performance reason, such info is not collected for now. 126 127 The attributes available are defined in ATTRIBUTES constant. 128 """ 129 130 _LXC_VERSION = None 131 132 def __init__(self, container_path, name, attribute_values, src=None, 133 snapshot=False): 134 """Initialize an object of LXC container with given attribute values. 135 136 @param container_path: Directory that stores the container. 137 @param name: Name of the container. 138 @param attribute_values: A dictionary of attribute values for the 139 container. 140 @param src: An optional source container. If provided, the source 141 continer is cloned, and the new container will point to the 142 clone. 143 @param snapshot: If a source container was specified, this argument 144 specifies whether or not to create a snapshot clone. 145 The default is to attempt to create a snapshot. 146 If a snapshot is requested and creating the snapshot 147 fails, a full clone will be attempted. 148 """ 149 self.container_path = os.path.realpath(container_path) 150 # Path to the rootfs of the container. This will be initialized when 151 # property rootfs is retrieved. 152 self._rootfs = None 153 self.name = name 154 for attribute, value in attribute_values.iteritems(): 155 setattr(self, attribute, value) 156 157 # Clone the container 158 if src is not None: 159 # Clone the source container to initialize this one. 160 lxc_utils.clone(src.container_path, src.name, self.container_path, 161 self.name, snapshot) 162 # Newly cloned containers have no ID. 163 self._id = None 164 else: 165 # This may be an existing container. Try to read the ID. 166 try: 167 self._id = ContainerId.load( 168 os.path.join(self.container_path, self.name)) 169 except (ValueError, TypeError): 170 # Ignore load errors. ContainerBucket currently queries every 171 # container quite frequently, and emitting exceptions here would 172 # cause any invalid containers on a server to block all 173 # ContainerBucket.get_all calls (see crbug/783865). 174 # TODO(kenobi): Containers with invalid ID files are probably 175 # the result of an aborted or failed operation. There is a 176 # non-zero chance that such containers would contain leftover 177 # state, or themselves be corrupted or invalid. Should we 178 # provide APIs for checking if a container is in this state? 179 logging.exception('Error loading ID for container %s:', 180 self.name) 181 self._id = None 182 183 if not Container._LXC_VERSION: 184 Container._LXC_VERSION = lxc_utils.get_lxc_version() 185 186 187 @classmethod 188 def create_from_existing_dir(cls, lxc_path, name, **kwargs): 189 """Creates a new container instance for an lxc container that already 190 exists on disk. 191 192 @param lxc_path: The LXC path for the container. 193 @param name: The container name. 194 195 @raise error.ContainerError: If the container doesn't already exist. 196 197 @return: The new container. 198 """ 199 return cls(lxc_path, name, kwargs) 200 201 202 # Containers have a name and an ID. The name is simply the name of the LXC 203 # container. The ID is the actual key that is used to identify the 204 # container to the autoserv system. In the case of a JIT-created container, 205 # we have the ID at the container's creation time so we use that to name the 206 # container. This may not be the case for other types of containers. 207 @classmethod 208 def clone(cls, src, new_name=None, new_path=None, snapshot=False, 209 cleanup=False): 210 """Creates a clone of this container. 211 212 @param src: The original container. 213 @param new_name: Name for the cloned container. If this is not 214 provided, a random unique container name will be 215 generated. 216 @param new_path: LXC path for the cloned container (optional; if not 217 specified, the new container is created in the same 218 directory as the source container). 219 @param snapshot: Whether to snapshot, or create a full clone. Note that 220 snapshot cloning is not supported on all platforms. If 221 this code is running on a platform that does not 222 support snapshot clones, this flag is ignored. 223 @param cleanup: If a container with the given name and path already 224 exist, clean it up first. 225 """ 226 if new_path is None: 227 new_path = src.container_path 228 229 if new_name is None: 230 _, new_name = os.path.split( 231 tempfile.mkdtemp(dir=new_path, prefix='container.')) 232 logging.debug('Generating new name for container: %s', new_name) 233 else: 234 # If a container exists at this location, clean it up first 235 container_folder = os.path.join(new_path, new_name) 236 if lxc_utils.path_exists(container_folder): 237 if not cleanup: 238 raise error.ContainerError('Container %s already exists.' % 239 new_name) 240 container = Container.create_from_existing_dir(new_path, 241 new_name) 242 try: 243 container.destroy() 244 except error.CmdError as e: 245 # The container could be created in a incompleted 246 # state. Delete the container folder instead. 247 logging.warn('Failed to destroy container %s, error: %s', 248 new_name, e) 249 utils.run('sudo rm -rf "%s"' % container_folder) 250 # Create the directory prior to creating the new container. This 251 # puts the ownership of the container under the current process's 252 # user, rather than root. This is necessary to enable the 253 # ContainerId to serialize properly. 254 os.mkdir(container_folder) 255 256 # Create and return the new container. 257 new_container = cls(new_path, new_name, {}, src, snapshot) 258 259 return new_container 260 261 262 def refresh_status(self): 263 """Refresh the status information of the container. 264 """ 265 containers = lxc.get_container_info(self.container_path, name=self.name) 266 if not containers: 267 raise error.ContainerError( 268 'No container found in directory %s with name of %s.' % 269 (self.container_path, self.name)) 270 attribute_values = containers[0] 271 for attribute, value in attribute_values.iteritems(): 272 setattr(self, attribute, value) 273 274 275 @property 276 def rootfs(self): 277 """Path to the rootfs of the container. 278 279 This property returns the path to the rootfs of the container, that is, 280 the folder where the container stores its local files. It reads the 281 attribute lxc.rootfs from the config file of the container, e.g., 282 lxc.rootfs = /usr/local/autotest/containers/t4/rootfs 283 If the container is created with snapshot, the rootfs is a chain of 284 folders, separated by `:` and ordered by how the snapshot is created, 285 e.g., 286 lxc.rootfs = overlayfs:/usr/local/autotest/containers/base/rootfs: 287 /usr/local/autotest/containers/t4_s/delta0 288 This function returns the last folder in the chain, in above example, 289 that is `/usr/local/autotest/containers/t4_s/delta0` 290 291 Files in the rootfs will be accessible directly within container. For 292 example, a folder in host "[rootfs]/usr/local/file1", can be accessed 293 inside container by path "/usr/local/file1". Note that symlink in the 294 host can not across host/container boundary, instead, directory mount 295 should be used, refer to function mount_dir. 296 297 @return: Path to the rootfs of the container. 298 """ 299 lxc_rootfs_config_name = 'lxc.rootfs' 300 # Check to see if the major lxc version is 3 or greater 301 if Container._LXC_VERSION: 302 logging.info("Detected lxc version %s", Container._LXC_VERSION) 303 if Container._LXC_VERSION[0] >= 3: 304 lxc_rootfs_config_name = 'lxc.rootfs.path' 305 if not self._rootfs: 306 lxc_rootfs = self._get_lxc_config(lxc_rootfs_config_name)[0] 307 cloned_from_snapshot = ':' in lxc_rootfs 308 if cloned_from_snapshot: 309 self._rootfs = lxc_rootfs.split(':')[-1] 310 else: 311 self._rootfs = lxc_rootfs 312 return self._rootfs 313 314 315 def attach_run(self, command, bash=True): 316 """Attach to a given container and run the given command. 317 318 @param command: Command to run in the container. 319 @param bash: Run the command through bash -c "command". This allows 320 pipes to be used in command. Default is set to True. 321 322 @return: The output of the command. 323 324 @raise error.CmdError: If container does not exist, or not running. 325 """ 326 cmd = 'sudo lxc-attach -P %s -n %s' % (self.container_path, self.name) 327 if bash and not command.startswith('bash -c'): 328 command = 'bash -c "%s"' % utils.sh_escape(command) 329 cmd += ' -- %s' % command 330 # TODO(dshi): crbug.com/459344 Set sudo to default to False when test 331 # container can be unprivileged container. 332 return utils.run(cmd) 333 334 335 def is_network_up(self): 336 """Check if network is up in the container by curl base container url. 337 338 @return: True if the network is up, otherwise False. 339 """ 340 try: 341 self.attach_run('curl --head %s' % constants.CONTAINER_BASE_URL) 342 return True 343 except error.CmdError as e: 344 logging.debug(e) 345 return False 346 347 348 @metrics.SecondsTimerDecorator( 349 '%s/container_start_duration' % constants.STATS_KEY) 350 def start(self, wait_for_network=True): 351 """Start the container. 352 353 @param wait_for_network: True to wait for network to be up. Default is 354 set to True. 355 356 @raise ContainerError: If container does not exist, or fails to start. 357 """ 358 cmd = 'sudo lxc-start -P %s -n %s -d' % (self.container_path, self.name) 359 output = utils.run(cmd).stdout 360 if not self.is_running(): 361 raise error.ContainerError( 362 'Container %s failed to start. lxc command output:\n%s' % 363 (os.path.join(self.container_path, self.name), 364 output)) 365 366 if wait_for_network: 367 logging.debug('Wait for network to be up.') 368 start_time = time.time() 369 utils.poll_for_condition( 370 condition=self.is_network_up, 371 timeout=constants.NETWORK_INIT_TIMEOUT, 372 sleep_interval=constants.NETWORK_INIT_CHECK_INTERVAL) 373 logging.debug('Network is up after %.2f seconds.', 374 time.time() - start_time) 375 376 377 @metrics.SecondsTimerDecorator( 378 '%s/container_stop_duration' % constants.STATS_KEY) 379 def stop(self): 380 """Stop the container. 381 382 @raise ContainerError: If container does not exist, or fails to start. 383 """ 384 cmd = 'sudo lxc-stop -P %s -n %s' % (self.container_path, self.name) 385 output = utils.run(cmd).stdout 386 self.refresh_status() 387 if self.state != 'STOPPED': 388 raise error.ContainerError( 389 'Container %s failed to be stopped. lxc command output:\n' 390 '%s' % (os.path.join(self.container_path, self.name), 391 output)) 392 393 394 @metrics.SecondsTimerDecorator( 395 '%s/container_destroy_duration' % constants.STATS_KEY) 396 def destroy(self, force=True): 397 """Destroy the container. 398 399 @param force: Set to True to force to destroy the container even if it's 400 running. This is faster than stop a container first then 401 try to destroy it. Default is set to True. 402 403 @raise ContainerError: If container does not exist or failed to destroy 404 the container. 405 """ 406 logging.debug('Destroying container %s/%s', 407 self.container_path, 408 self.name) 409 cmd = 'sudo lxc-destroy -P %s -n %s' % (self.container_path, 410 self.name) 411 if force: 412 cmd += ' -f' 413 utils.run(cmd) 414 415 416 def mount_dir(self, source, destination, readonly=False): 417 """Mount a directory in host to a directory in the container. 418 419 @param source: Directory in host to be mounted. 420 @param destination: Directory in container to mount the source directory 421 @param readonly: Set to True to make a readonly mount, default is False. 422 """ 423 # Destination path in container must be relative. 424 destination = destination.lstrip('/') 425 # Create directory in container for mount. Changes to container rootfs 426 # require sudo. 427 utils.run('sudo mkdir -p %s' % os.path.join(self.rootfs, destination)) 428 mount = ('%s %s none bind%s 0 0' % 429 (source, destination, ',ro' if readonly else '')) 430 self._set_lxc_config('lxc.mount.entry', mount) 431 432 def verify_autotest_setup(self, job_folder): 433 """Verify autotest code is set up properly in the container. 434 435 @param job_folder: Name of the job result folder. 436 437 @raise ContainerError: If autotest code is not set up properly. 438 """ 439 # Test autotest code is setup by verifying a list of 440 # (directory, minimum file count) 441 directories_to_check = [ 442 (constants.CONTAINER_AUTOTEST_DIR, 3), 443 (constants.RESULT_DIR_FMT % job_folder, 0), 444 (constants.CONTAINER_SITE_PACKAGES_PATH, 3)] 445 for directory, count in directories_to_check: 446 result = self.attach_run(command=(constants.COUNT_FILE_CMD % 447 {'dir': directory})).stdout 448 logging.debug('%s entries in %s.', int(result), directory) 449 if int(result) < count: 450 raise error.ContainerError('%s is not properly set up.' % 451 directory) 452 # lxc-attach and run command does not run in shell, thus .bashrc is not 453 # loaded. Following command creates a symlink in /usr/bin/ for gsutil 454 # if it's installed. 455 # TODO(dshi): Remove this code after lab container is updated with 456 # gsutil installed in /usr/bin/ 457 self.attach_run('test -f /root/gsutil/gsutil && ' 458 'ln -s /root/gsutil/gsutil /usr/bin/gsutil || true') 459 460 461 def modify_import_order(self): 462 """Swap the python import order of lib and local/lib. 463 464 In Moblab, the host's python modules located in 465 /usr/lib64/python2.7/site-packages is mounted to following folder inside 466 container: /usr/local/lib/python2.7/dist-packages/. The modules include 467 an old version of requests module, which is used in autotest 468 site-packages. For test, the module is only used in 469 dev_server/symbolicate_dump for requests.call and requests.codes.OK. 470 When pip is installed inside the container, it installs requests module 471 with version of 2.2.1 in /usr/lib/python2.7/dist-packages/. The version 472 is newer than the one used in autotest site-packages, but not the latest 473 either. 474 According to /usr/lib/python2.7/site.py, modules in /usr/local/lib are 475 imported before the ones in /usr/lib. That leads to pip to use the older 476 version of requests (0.11.2), and it will fail. On the other hand, 477 requests module 2.2.1 can't be installed in CrOS (refer to CL:265759), 478 and higher version of requests module can't work with pip. 479 The only fix to resolve this is to switch the import order, so modules 480 in /usr/lib can be imported before /usr/local/lib. 481 """ 482 site_module = '/usr/lib/python2.7/site.py' 483 self.attach_run("sed -i ':a;N;$!ba;s/\"local\/lib\",\\n/" 484 "\"lib_placeholder\",\\n/g' %s" % site_module) 485 self.attach_run("sed -i ':a;N;$!ba;s/\"lib\",\\n/" 486 "\"local\/lib\",\\n/g' %s" % site_module) 487 self.attach_run('sed -i "s/lib_placeholder/lib/g" %s' % 488 site_module) 489 490 491 def is_running(self): 492 """Returns whether or not this container is currently running.""" 493 self.refresh_status() 494 return self.state == 'RUNNING' 495 496 497 def set_hostname(self, hostname): 498 """Sets the hostname within the container. 499 500 This method can only be called on a running container. 501 502 @param hostname The new container hostname. 503 504 @raise ContainerError: If the container is not running. 505 """ 506 if not self.is_running(): 507 raise error.ContainerError( 508 'set_hostname can only be called on running containers.') 509 510 self.attach_run('hostname %s' % (hostname)) 511 self.attach_run(constants.APPEND_CMD_FMT % { 512 'content': '127.0.0.1 %s' % (hostname), 513 'file': '/etc/hosts'}) 514 515 516 def install_ssp(self, ssp_url): 517 """Downloads and installs the given server package. 518 519 @param ssp_url: The URL of the ssp to download and install. 520 """ 521 usr_local_path = os.path.join(self.rootfs, 'usr', 'local') 522 autotest_pkg_path = os.path.join(usr_local_path, 523 'autotest_server_package.tar.bz2') 524 # Changes within the container rootfs require sudo. 525 utils.run('sudo mkdir -p %s'% usr_local_path) 526 527 lxc.download_extract(ssp_url, autotest_pkg_path, usr_local_path) 528 529 def install_ssp_isolate(self, isolate_hash, dest_path=None): 530 """Downloads and install the contents of the given isolate. 531 This places the isolate contents under /usr/local or a provided path. 532 Most commonly this is a copy of a specific autotest version, in which 533 case: 534 /usr/local/autotest contains the autotest code 535 /usr/local/logs contains logs from the installation process. 536 537 @param isolate_hash: The hash string which serves as a key to retrieve 538 the desired isolate 539 @param dest_path: Path to the directory to place the isolate in. 540 Defaults to /usr/local/ 541 542 @return: Exit status of the installation command. 543 """ 544 dest_path = dest_path or os.path.join(self.rootfs, 'usr', 'local') 545 isolate_log_path = os.path.join( 546 self.rootfs, 'usr', 'local', 'logs', 'isolate') 547 log_file = os.path.join(isolate_log_path, 548 'contents.' + time.strftime('%Y-%m-%d-%H.%M.%S')) 549 550 utils.run('sudo mkdir -p %s' % isolate_log_path) 551 _command = ("sudo isolated download -isolated {sha} -I {server}" 552 " -output-dir {dest_dir} -output-files {log_file}") 553 554 return utils.run(_command.format( 555 sha=isolate_hash, dest_dir=dest_path, 556 log_file=log_file, server=chromite_constants.ISOLATESERVER)) 557 558 559 def install_control_file(self, control_file): 560 """Installs the given control file. 561 562 The given file will be copied into the container. 563 564 @param control_file: Path to the control file to install. 565 """ 566 dst = os.path.join(constants.CONTROL_TEMP_PATH, 567 os.path.basename(control_file)) 568 self.copy(control_file, dst) 569 570 571 def copy(self, host_path, container_path): 572 """Copies files into the container. 573 574 @param host_path: Path to the source file/dir to be copied. 575 @param container_path: Path to the destination dir (in the container). 576 """ 577 dst_path = os.path.join(self.rootfs, 578 container_path.lstrip(os.path.sep)) 579 self._do_copy(src=host_path, dst=dst_path) 580 581 582 @property 583 def id(self): 584 """Returns the container ID.""" 585 return self._id 586 587 588 @id.setter 589 def id(self, new_id): 590 """Sets the container ID.""" 591 self._id = new_id; 592 # Persist the ID so other container objects can pick it up. 593 self._id.save(os.path.join(self.container_path, self.name)) 594 595 596 def _do_copy(self, src, dst): 597 """Copies files and directories on the host system. 598 599 @param src: The source file or directory. 600 @param dst: The destination file or directory. If the path to the 601 destination does not exist, it will be created. 602 """ 603 # Create the dst dir. mkdir -p will not fail if dst_dir exists. 604 dst_dir = os.path.dirname(dst) 605 # Make sure the source ends with `/.` if it's a directory. Otherwise 606 # command cp will not work. 607 if os.path.isdir(src) and os.path.split(src)[1] != '.': 608 src = os.path.join(src, '.') 609 utils.run("sudo sh -c 'mkdir -p \"%s\" && cp -RL \"%s\" \"%s\"'" % 610 (dst_dir, src, dst)) 611 612 def _set_lxc_config(self, key, value): 613 """Sets an LXC config value for this container. 614 615 Configuration changes made while a container is running don't take 616 effect until the container is restarted. Since this isn't a scenario 617 that should ever come up in our use cases, calling this method on a 618 running container will cause a ContainerError. 619 620 @param key: The LXC config key to set. 621 @param value: The value to use for the given key. 622 623 @raise error.ContainerError: If the container is already started. 624 """ 625 if self.is_running(): 626 raise error.ContainerError( 627 '_set_lxc_config(%s, %s) called on a running container.' % 628 (key, value)) 629 config_file = os.path.join(self.container_path, self.name, 'config') 630 config = '%s = %s' % (key, value) 631 utils.run( 632 constants.APPEND_CMD_FMT % {'content': config, 'file': config_file}) 633 634 635 def _get_lxc_config(self, key): 636 """Retrieves an LXC config value from the container. 637 638 @param key The key of the config value to retrieve. 639 """ 640 cmd = ('sudo lxc-info -P %s -n %s -c %s' % 641 (self.container_path, self.name, key)) 642 config = utils.run(cmd).stdout.strip().splitlines() 643 644 # Strip the decoration from line 1 of the output. 645 match = re.match('%s = (.*)' % key, config[0]) 646 if not match: 647 raise error.ContainerError( 648 'Config %s not found for container %s. (%s)' % 649 (key, self.name, ','.join(config))) 650 config[0] = match.group(1) 651 return config 652