1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import logging 6import os 7import re 8import sys 9import time 10 11import common 12from autotest_lib.client.bin import utils 13from autotest_lib.client.common_lib import autotemp 14from autotest_lib.client.common_lib import error 15from autotest_lib.client.common_lib import global_config 16from autotest_lib.client.common_lib import hosts 17from autotest_lib.client.common_lib import lsbrelease_utils 18from autotest_lib.client.common_lib.cros import autoupdater 19from autotest_lib.client.common_lib.cros import dev_server 20from autotest_lib.client.common_lib.cros import retry 21from autotest_lib.client.cros import constants as client_constants 22from autotest_lib.client.cros import cros_ui 23from autotest_lib.client.cros.audio import cras_utils 24from autotest_lib.client.cros.input_playback import input_playback 25from autotest_lib.client.cros.video import constants as video_test_constants 26from autotest_lib.server import afe_utils 27from autotest_lib.server import autoserv_parser 28from autotest_lib.server import autotest 29from autotest_lib.server import constants 30from autotest_lib.server import utils as server_utils 31from autotest_lib.server.cros import provision 32from autotest_lib.server.cros.dynamic_suite import constants as ds_constants 33from autotest_lib.server.cros.dynamic_suite import tools, frontend_wrappers 34from autotest_lib.server.cros.faft.config.config import Config as FAFTConfig 35from autotest_lib.server.cros.servo import firmware_programmer 36from autotest_lib.server.cros.servo import plankton 37from autotest_lib.server.hosts import abstract_ssh 38from autotest_lib.server.hosts import base_label 39from autotest_lib.server.hosts import cros_label 40from autotest_lib.server.hosts import chameleon_host 41from autotest_lib.server.hosts import cros_repair 42from autotest_lib.server.hosts import plankton_host 43from autotest_lib.server.hosts import servo_host 44from autotest_lib.site_utils.rpm_control_system import rpm_client 45 46# In case cros_host is being ran via SSP on an older Moblab version with an 47# older chromite version. 48try: 49 from chromite.lib import metrics 50except ImportError: 51 metrics = utils.metrics_mock 52 53 54CONFIG = global_config.global_config 55ENABLE_DEVSERVER_TRIGGER_AUTO_UPDATE = CONFIG.get_config_value( 56 'CROS', 'enable_devserver_trigger_auto_update', type=bool, 57 default=False) 58 59 60class FactoryImageCheckerException(error.AutoservError): 61 """Exception raised when an image is a factory image.""" 62 pass 63 64 65class CrosHost(abstract_ssh.AbstractSSHHost): 66 """Chromium OS specific subclass of Host.""" 67 68 VERSION_PREFIX = provision.CROS_VERSION_PREFIX 69 70 _parser = autoserv_parser.autoserv_parser 71 _AFE = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10) 72 support_devserver_provision = ENABLE_DEVSERVER_TRIGGER_AUTO_UPDATE 73 74 # Timeout values (in seconds) associated with various Chrome OS 75 # state changes. 76 # 77 # In general, a good rule of thumb is that the timeout can be up 78 # to twice the typical measured value on the slowest platform. 79 # The times here have not necessarily been empirically tested to 80 # meet this criterion. 81 # 82 # SLEEP_TIMEOUT: Time to allow for suspend to memory. 83 # RESUME_TIMEOUT: Time to allow for resume after suspend, plus 84 # time to restart the netwowrk. 85 # SHUTDOWN_TIMEOUT: Time to allow for shut down. 86 # BOOT_TIMEOUT: Time to allow for boot from power off. Among 87 # other things, this must account for the 30 second dev-mode 88 # screen delay, time to start the network on the DUT, and the 89 # ssh timeout of 120 seconds. 90 # USB_BOOT_TIMEOUT: Time to allow for boot from a USB device, 91 # including the 30 second dev-mode delay and time to start the 92 # network. 93 # INSTALL_TIMEOUT: Time to allow for chromeos-install. 94 # POWERWASH_BOOT_TIMEOUT: Time to allow for a reboot that 95 # includes powerwash. 96 97 SLEEP_TIMEOUT = 2 98 RESUME_TIMEOUT = 10 99 SHUTDOWN_TIMEOUT = 10 100 BOOT_TIMEOUT = 150 101 USB_BOOT_TIMEOUT = 300 102 INSTALL_TIMEOUT = 480 103 POWERWASH_BOOT_TIMEOUT = 60 104 105 # Minimum OS version that supports server side packaging. Older builds may 106 # not have server side package built or with Autotest code change to support 107 # server-side packaging. 108 MIN_VERSION_SUPPORT_SSP = CONFIG.get_config_value( 109 'AUTOSERV', 'min_version_support_ssp', type=int) 110 111 # REBOOT_TIMEOUT: How long to wait for a reboot. 112 # 113 # We have a long timeout to ensure we don't flakily fail due to other 114 # issues. Shorter timeouts are vetted in platform_RebootAfterUpdate. 115 # TODO(sbasi - crbug.com/276094) Restore to 5 mins once the 'host did not 116 # return from reboot' bug is solved. 117 REBOOT_TIMEOUT = 480 118 119 # _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF. 120 # _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle. 121 _USB_POWER_TIMEOUT = 5 122 _POWER_CYCLE_TIMEOUT = 10 123 124 _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine' 125 _RPM_HOSTNAME_REGEX = ('chromeos(\d+)(-row(\d+))?-rack(\d+[a-z]*)' 126 '-host(\d+)') 127 _LIGHTSENSOR_FILES = [ "in_illuminance0_input", 128 "in_illuminance_input", 129 "in_illuminance0_raw", 130 "in_illuminance_raw", 131 "illuminance0_input"] 132 _LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices' 133 134 # Constants used in ping_wait_up() and ping_wait_down(). 135 # 136 # _PING_WAIT_COUNT is the approximate number of polling 137 # cycles to use when waiting for a host state change. 138 # 139 # _PING_STATUS_DOWN and _PING_STATUS_UP are names used 140 # for arguments to the internal _ping_wait_for_status() 141 # method. 142 _PING_WAIT_COUNT = 40 143 _PING_STATUS_DOWN = False 144 _PING_STATUS_UP = True 145 146 # Allowed values for the power_method argument. 147 148 # POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods. 149 # POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods. 150 # POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods. 151 POWER_CONTROL_RPM = 'RPM' 152 POWER_CONTROL_SERVO = 'servoj10' 153 POWER_CONTROL_MANUAL = 'manual' 154 155 POWER_CONTROL_VALID_ARGS = (POWER_CONTROL_RPM, 156 POWER_CONTROL_SERVO, 157 POWER_CONTROL_MANUAL) 158 159 _RPM_OUTLET_CHANGED = 'outlet_changed' 160 161 # URL pattern to download firmware image. 162 _FW_IMAGE_URL_PATTERN = CONFIG.get_config_value( 163 'CROS', 'firmware_url_pattern', type=str) 164 165 166 # A flag file to indicate provision failures. The file is created 167 # at the start of any AU procedure (see `machine_install()`). The 168 # file's location in stateful means that on successul update it will 169 # be removed. Thus, if this file exists, it indicates that we've 170 # tried and failed in a previous attempt to update. 171 PROVISION_FAILED = '/var/tmp/provision_failed' 172 173 174 @staticmethod 175 def check_host(host, timeout=10): 176 """ 177 Check if the given host is a chrome-os host. 178 179 @param host: An ssh host representing a device. 180 @param timeout: The timeout for the run command. 181 182 @return: True if the host device is chromeos. 183 184 """ 185 try: 186 result = host.run( 187 'grep -q CHROMEOS /etc/lsb-release && ' 188 '! test -f /mnt/stateful_partition/.android_tester && ' 189 '! grep -q moblab /etc/lsb-release', 190 ignore_status=True, timeout=timeout) 191 if result.exit_status == 0: 192 lsb_release_content = host.run( 193 'grep CHROMEOS_RELEASE_BOARD /etc/lsb-release', 194 timeout=timeout).stdout 195 return not ( 196 lsbrelease_utils.is_jetstream( 197 lsb_release_content=lsb_release_content) or 198 lsbrelease_utils.is_gce_board( 199 lsb_release_content=lsb_release_content)) 200 201 except (error.AutoservRunError, error.AutoservSSHTimeout): 202 return False 203 204 return False 205 206 207 @staticmethod 208 def get_chameleon_arguments(args_dict): 209 """Extract chameleon options from `args_dict` and return the result. 210 211 Recommended usage: 212 ~~~~~~~~ 213 args_dict = utils.args_to_dict(args) 214 chameleon_args = hosts.CrosHost.get_chameleon_arguments(args_dict) 215 host = hosts.create_host(machine, chameleon_args=chameleon_args) 216 ~~~~~~~~ 217 218 @param args_dict Dictionary from which to extract the chameleon 219 arguments. 220 """ 221 return {key: args_dict[key] 222 for key in ('chameleon_host', 'chameleon_port') 223 if key in args_dict} 224 225 226 @staticmethod 227 def get_plankton_arguments(args_dict): 228 """Extract chameleon options from `args_dict` and return the result. 229 230 Recommended usage: 231 ~~~~~~~~ 232 args_dict = utils.args_to_dict(args) 233 plankton_args = hosts.CrosHost.get_plankton_arguments(args_dict) 234 host = hosts.create_host(machine, plankton_args=plankton_args) 235 ~~~~~~~~ 236 237 @param args_dict Dictionary from which to extract the plankton 238 arguments. 239 """ 240 return {key: args_dict[key] 241 for key in ('plankton_host', 'plankton_port') 242 if key in args_dict} 243 244 245 @staticmethod 246 def get_servo_arguments(args_dict): 247 """Extract servo options from `args_dict` and return the result. 248 249 Recommended usage: 250 ~~~~~~~~ 251 args_dict = utils.args_to_dict(args) 252 servo_args = hosts.CrosHost.get_servo_arguments(args_dict) 253 host = hosts.create_host(machine, servo_args=servo_args) 254 ~~~~~~~~ 255 256 @param args_dict Dictionary from which to extract the servo 257 arguments. 258 """ 259 servo_attrs = (servo_host.SERVO_HOST_ATTR, 260 servo_host.SERVO_PORT_ATTR, 261 servo_host.SERVO_BOARD_ATTR) 262 servo_args = {key: args_dict[key] 263 for key in servo_attrs 264 if key in args_dict} 265 return ( 266 None 267 if servo_host.SERVO_HOST_ATTR in servo_args 268 and not servo_args[servo_host.SERVO_HOST_ATTR] 269 else servo_args) 270 271 272 def _initialize(self, hostname, chameleon_args=None, servo_args=None, 273 plankton_args=None, try_lab_servo=False, 274 try_servo_repair=False, 275 ssh_verbosity_flag='', ssh_options='', 276 *args, **dargs): 277 """Initialize superclasses, |self.chameleon|, and |self.servo|. 278 279 This method will attempt to create the test-assistant object 280 (chameleon/servo) when it is needed by the test. Check 281 the docstring of chameleon_host.create_chameleon_host and 282 servo_host.create_servo_host for how this is determined. 283 284 @param hostname: Hostname of the dut. 285 @param chameleon_args: A dictionary that contains args for creating 286 a ChameleonHost. See chameleon_host for details. 287 @param servo_args: A dictionary that contains args for creating 288 a ServoHost object. See servo_host for details. 289 @param try_lab_servo: When true, indicates that an attempt should 290 be made to create a ServoHost for a DUT in 291 the test lab, even if not required by 292 `servo_args`. See servo_host for details. 293 @param try_servo_repair: If a servo host is created, check it 294 with `repair()` rather than `verify()`. 295 See servo_host for details. 296 @param ssh_verbosity_flag: String, to pass to the ssh command to control 297 verbosity. 298 @param ssh_options: String, other ssh options to pass to the ssh 299 command. 300 """ 301 super(CrosHost, self)._initialize(hostname=hostname, 302 *args, **dargs) 303 self._repair_strategy = cros_repair.create_cros_repair_strategy() 304 self.labels = base_label.LabelRetriever(cros_label.CROS_LABELS) 305 # self.env is a dictionary of environment variable settings 306 # to be exported for commands run on the host. 307 # LIBC_FATAL_STDERR_ can be useful for diagnosing certain 308 # errors that might happen. 309 self.env['LIBC_FATAL_STDERR_'] = '1' 310 self._ssh_verbosity_flag = ssh_verbosity_flag 311 self._ssh_options = ssh_options 312 # TODO(fdeng): We need to simplify the 313 # process of servo and servo_host initialization. 314 # crbug.com/298432 315 self._servo_host = servo_host.create_servo_host( 316 dut=self, servo_args=servo_args, 317 try_lab_servo=try_lab_servo, 318 try_servo_repair=try_servo_repair) 319 if self._servo_host is not None: 320 self.servo = self._servo_host.get_servo() 321 else: 322 self.servo = None 323 324 # TODO(waihong): Do the simplication on Chameleon too. 325 self._chameleon_host = chameleon_host.create_chameleon_host( 326 dut=self.hostname, chameleon_args=chameleon_args) 327 # Add plankton host if plankton args were added on command line 328 self._plankton_host = plankton_host.create_plankton_host(plankton_args) 329 330 if self._chameleon_host: 331 self.chameleon = self._chameleon_host.create_chameleon_board() 332 else: 333 self.chameleon = None 334 335 if self._plankton_host: 336 self.plankton_servo = self._plankton_host.get_servo() 337 logging.info('plankton_servo: %r', self.plankton_servo) 338 # Create the plankton object used to access the ec uart 339 self.plankton = plankton.Plankton(self.plankton_servo, 340 self._plankton_host.get_servod_server_proxy()) 341 else: 342 self.plankton = None 343 344 345 def _get_cros_repair_image_name(self): 346 info = self.host_info_store.get() 347 if info.board is None: 348 raise error.AutoservError('Cannot obtain repair image name. ' 349 'No board label value found') 350 351 return afe_utils.get_stable_cros_image_name(info.board) 352 353 354 def host_version_prefix(self, image): 355 """Return version label prefix. 356 357 In case the CrOS provisioning version is something other than the 358 standard CrOS version e.g. CrOS TH version, this function will 359 find the prefix from provision.py. 360 361 @param image: The image name to find its version prefix. 362 @returns: A prefix string for the image type. 363 """ 364 return provision.get_version_label_prefix(image) 365 366 367 def verify_job_repo_url(self, tag=''): 368 """ 369 Make sure job_repo_url of this host is valid. 370 371 Eg: The job_repo_url "http://lmn.cd.ab.xyx:8080/static/\ 372 lumpy-release/R29-4279.0.0/autotest/packages" claims to have the 373 autotest package for lumpy-release/R29-4279.0.0. If this isn't the case, 374 download and extract it. If the devserver embedded in the url is 375 unresponsive, update the job_repo_url of the host after staging it on 376 another devserver. 377 378 @param job_repo_url: A url pointing to the devserver where the autotest 379 package for this build should be staged. 380 @param tag: The tag from the server job, in the format 381 <job_id>-<user>/<hostname>, or <hostless> for a server job. 382 383 @raises DevServerException: If we could not resolve a devserver. 384 @raises AutoservError: If we're unable to save the new job_repo_url as 385 a result of choosing a new devserver because the old one failed to 386 respond to a health check. 387 @raises urllib2.URLError: If the devserver embedded in job_repo_url 388 doesn't respond within the timeout. 389 """ 390 info = self.host_info_store.get() 391 job_repo_url = info.attributes.get(ds_constants.JOB_REPO_URL, '') 392 if not job_repo_url: 393 logging.warning('No job repo url set on host %s', self.hostname) 394 return 395 396 logging.info('Verifying job repo url %s', job_repo_url) 397 devserver_url, image_name = tools.get_devserver_build_from_package_url( 398 job_repo_url) 399 400 ds = dev_server.ImageServer(devserver_url) 401 402 logging.info('Staging autotest artifacts for %s on devserver %s', 403 image_name, ds.url()) 404 405 start_time = time.time() 406 ds.stage_artifacts(image_name, ['autotest_packages']) 407 stage_time = time.time() - start_time 408 409 # Record how much of the verification time comes from a devserver 410 # restage. If we're doing things right we should not see multiple 411 # devservers for a given board/build/branch path. 412 try: 413 board, build_type, branch = server_utils.ParseBuildName( 414 image_name)[:3] 415 except server_utils.ParseBuildNameException: 416 pass 417 else: 418 devserver = devserver_url[ 419 devserver_url.find('/') + 2:devserver_url.rfind(':')] 420 stats_key = { 421 'board': board, 422 'build_type': build_type, 423 'branch': branch, 424 'devserver': devserver.replace('.', '_'), 425 } 426 427 monarch_fields = { 428 'board': board, 429 'build_type': build_type, 430 # TODO(akeshet): To be consistent with most other metrics, 431 # consider changing the following field to be named 432 # 'milestone'. 433 'branch': branch, 434 'dev_server': devserver, 435 } 436 metrics.Counter( 437 'chromeos/autotest/provision/verify_url' 438 ).increment(fields=monarch_fields) 439 metrics.SecondsDistribution( 440 'chromeos/autotest/provision/verify_url_duration' 441 ).add(stage_time, fields=monarch_fields) 442 443 444 def stage_server_side_package(self, image=None): 445 """Stage autotest server-side package on devserver. 446 447 @param image: Full path of an OS image to install or a build name. 448 449 @return: A url to the autotest server-side package. 450 451 @raise: error.AutoservError if fail to locate the build to test with, or 452 fail to stage server-side package. 453 """ 454 # If enable_drone_in_restricted_subnet is False, do not set hostname 455 # in devserver.resolve call, so a devserver in non-restricted subnet 456 # is picked to stage autotest server package for drone to download. 457 hostname = self.hostname 458 if not server_utils.ENABLE_DRONE_IN_RESTRICTED_SUBNET: 459 hostname = None 460 if image: 461 image_name = tools.get_build_from_image(image) 462 if not image_name: 463 raise error.AutoservError( 464 'Failed to parse build name from %s' % image) 465 ds = dev_server.ImageServer.resolve(image_name, hostname) 466 else: 467 info = self.host_info_store.get() 468 job_repo_url = info.attributes.get(ds_constants.JOB_REPO_URL, '') 469 if job_repo_url: 470 devserver_url, image_name = ( 471 tools.get_devserver_build_from_package_url(job_repo_url)) 472 # If enable_drone_in_restricted_subnet is True, use the 473 # existing devserver. Otherwise, resolve a new one in 474 # non-restricted subnet. 475 if server_utils.ENABLE_DRONE_IN_RESTRICTED_SUBNET: 476 ds = dev_server.ImageServer(devserver_url) 477 else: 478 ds = dev_server.ImageServer.resolve(image_name) 479 elif info.build is not None: 480 ds = dev_server.ImageServer.resolve(info.build, hostname) 481 image_name = info.build 482 else: 483 raise error.AutoservError( 484 'Failed to stage server-side package. The host has ' 485 'no job_report_url attribute or version label.') 486 487 # Get the OS version of the build, for any build older than 488 # MIN_VERSION_SUPPORT_SSP, server side packaging is not supported. 489 match = re.match('.*/R\d+-(\d+)\.', image_name) 490 if match and int(match.group(1)) < self.MIN_VERSION_SUPPORT_SSP: 491 raise error.AutoservError( 492 'Build %s is older than %s. Server side packaging is ' 493 'disabled.' % (image_name, self.MIN_VERSION_SUPPORT_SSP)) 494 495 ds.stage_artifacts(image_name, ['autotest_server_package']) 496 return '%s/static/%s/%s' % (ds.url(), image_name, 497 'autotest_server_package.tar.bz2') 498 499 500 def _try_stateful_update(self, update_url, force_update, updater): 501 """Try to use stateful update to initialize DUT. 502 503 When DUT is already running the same version that machine_install 504 tries to install, stateful update is a much faster way to clean up 505 the DUT for testing, compared to a full reimage. It is implemeted 506 by calling autoupdater.run_update, but skipping updating root, as 507 updating the kernel is time consuming and not necessary. 508 509 @param update_url: url of the image. 510 @param force_update: Set to True to update the image even if the DUT 511 is running the same version. 512 @param updater: ChromiumOSUpdater instance used to update the DUT. 513 @returns: True if the DUT was updated with stateful update. 514 515 """ 516 self.prepare_for_update() 517 518 # TODO(jrbarnette): Yes, I hate this re.match() test case. 519 # It's better than the alternative: see crbug.com/360944. 520 image_name = autoupdater.url_to_image_name(update_url) 521 release_pattern = r'^.*-release/R[0-9]+-[0-9]+\.[0-9]+\.0$' 522 if not re.match(release_pattern, image_name): 523 return False 524 if not updater.check_version(): 525 return False 526 if not force_update: 527 logging.info('Canceling stateful update because the new and ' 528 'old versions are the same.') 529 return False 530 # Following folders should be rebuilt after stateful update. 531 # A test file is used to confirm each folder gets rebuilt after 532 # the stateful update. 533 folders_to_check = ['/var', '/home', '/mnt/stateful_partition'] 534 test_file = '.test_file_to_be_deleted' 535 paths = [os.path.join(folder, test_file) for folder in folders_to_check] 536 self.run('touch %s' % ' '.join(paths)) 537 538 updater.run_update(update_root=False) 539 540 # Reboot to complete stateful update. 541 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) 542 543 # After stateful update and a reboot, all of the test_files shouldn't 544 # exist any more. Otherwise the stateful update is failed. 545 return not any( 546 self.path_exists(os.path.join(folder, test_file)) 547 for folder in folders_to_check) 548 549 550 def _post_update_processing(self, updater, expected_kernel=None): 551 """After the DUT is updated, confirm machine_install succeeded. 552 553 @param updater: ChromiumOSUpdater instance used to update the DUT. 554 @param expected_kernel: kernel expected to be active after reboot, 555 or `None` to skip rollback checking. 556 557 """ 558 # Touch the lab machine file to leave a marker that 559 # distinguishes this image from other test images. 560 # Afterwards, we must re-run the autoreboot script because 561 # it depends on the _LAB_MACHINE_FILE. 562 autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || ' 563 '( touch "$FILE" ; start autoreboot )') 564 self.run(autoreboot_cmd % self._LAB_MACHINE_FILE) 565 updater.verify_boot_expectations( 566 expected_kernel, rollback_message= 567 'Build %s failed to boot on %s; system rolled back to previous ' 568 'build' % (updater.update_version, self.hostname)) 569 # Check that we've got the build we meant to install. 570 if not updater.check_version_to_confirm_install(): 571 raise autoupdater.ChromiumOSError( 572 'Failed to update %s to build %s; found build ' 573 '%s instead' % (self.hostname, 574 updater.update_version, 575 self.get_release_version())) 576 577 logging.debug('Cleaning up old autotest directories.') 578 try: 579 installed_autodir = autotest.Autotest.get_installed_autodir(self) 580 self.run('rm -rf ' + installed_autodir) 581 except autotest.AutodirNotFoundError: 582 logging.debug('No autotest installed directory found.') 583 584 585 def _stage_image_for_update(self, image_name=None): 586 """Stage a build on a devserver and return the update_url and devserver. 587 588 @param image_name: a name like lumpy-release/R27-3837.0.0 589 @returns a tuple with an update URL like: 590 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 591 and the devserver instance. 592 """ 593 if not image_name: 594 image_name = self._get_cros_repair_image_name() 595 logging.info('Staging build for AU: %s', image_name) 596 devserver = dev_server.ImageServer.resolve(image_name, self.hostname) 597 devserver.trigger_download(image_name, synchronous=False) 598 return (tools.image_url_pattern() % (devserver.url(), image_name), 599 devserver) 600 601 602 def stage_image_for_servo(self, image_name=None): 603 """Stage a build on a devserver and return the update_url. 604 605 @param image_name: a name like lumpy-release/R27-3837.0.0 606 @returns an update URL like: 607 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 608 """ 609 if not image_name: 610 image_name = self._get_cros_repair_image_name() 611 logging.info('Staging build for servo install: %s', image_name) 612 devserver = dev_server.ImageServer.resolve(image_name, self.hostname) 613 devserver.stage_artifacts(image_name, ['test_image']) 614 return devserver.get_test_image_url(image_name) 615 616 617 def stage_factory_image_for_servo(self, image_name): 618 """Stage a build on a devserver and return the update_url. 619 620 @param image_name: a name like <baord>/4262.204.0 621 622 @return: An update URL, eg: 623 http://<devserver>/static/canary-channel/\ 624 <board>/4262.204.0/factory_test/chromiumos_factory_image.bin 625 626 @raises: ValueError if the factory artifact name is missing from 627 the config. 628 629 """ 630 if not image_name: 631 logging.error('Need an image_name to stage a factory image.') 632 return 633 634 factory_artifact = CONFIG.get_config_value( 635 'CROS', 'factory_artifact', type=str, default='') 636 if not factory_artifact: 637 raise ValueError('Cannot retrieve the factory artifact name from ' 638 'autotest config, and hence cannot stage factory ' 639 'artifacts.') 640 641 logging.info('Staging build for servo install: %s', image_name) 642 devserver = dev_server.ImageServer.resolve(image_name, self.hostname) 643 devserver.stage_artifacts( 644 image_name, 645 [factory_artifact], 646 archive_url=None) 647 648 return tools.factory_image_url_pattern() % (devserver.url(), image_name) 649 650 651 def _get_au_monarch_fields(self, devserver, build): 652 """Form monarch fields by given devserve & build for auto-update. 653 654 @param devserver: the devserver (ImageServer instance) for auto-update. 655 @param build: the build to be updated. 656 657 @return A dictionary of monach fields. 658 """ 659 try: 660 board, build_type, milestone, _ = server_utils.ParseBuildName(build) 661 except server_utils.ParseBuildNameException: 662 logging.warning('Unable to parse build name %s. Something is ' 663 'likely broken, but will continue anyway.', 664 build) 665 board, build_type, milestone = ('', '', '') 666 667 monarch_fields = { 668 'dev_server': devserver.hostname, 669 'board': board, 670 'build_type': build_type, 671 'milestone': milestone 672 } 673 return monarch_fields 674 675 676 def _retry_auto_update_with_new_devserver(self, build, last_devserver, 677 force_update, force_full_update, 678 force_original, quick_provision): 679 """Kick off auto-update by devserver and send metrics. 680 681 @param build: the build to update. 682 @param last_devserver: the last devserver that failed to provision. 683 @param force_update: see |machine_install_by_devserver|'s force_udpate 684 for details. 685 @param force_full_update: see |machine_install_by_devserver|'s 686 force_full_update for details. 687 @param force_original: Whether to force stateful update with the 688 original payload. 689 @param quick_provision: Attempt to use quick provision path first. 690 691 @return the result of |auto_update| in dev_server. 692 """ 693 devserver = dev_server.resolve( 694 build, self.hostname, ban_list=[last_devserver.url()]) 695 devserver.trigger_download(build, synchronous=False) 696 monarch_fields = self._get_au_monarch_fields(devserver, build) 697 logging.debug('Retry auto_update: resolved devserver for ' 698 'auto-update: %s', devserver.url()) 699 700 # Add metrics 701 install_with_dev_counter = metrics.Counter( 702 'chromeos/autotest/provision/install_with_devserver') 703 install_with_dev_counter.increment(fields=monarch_fields) 704 c = metrics.Counter( 705 'chromeos/autotest/provision/retry_by_devserver') 706 monarch_fields['last_devserver'] = last_devserver.hostname 707 monarch_fields['host'] = self.hostname 708 c.increment(fields=monarch_fields) 709 710 # Won't retry auto_update in a retry of auto-update. 711 # In other words, we only retry auto-update once with a different 712 # devservers. 713 devserver.auto_update( 714 self.hostname, build, 715 original_board=self.get_board().replace( 716 ds_constants.BOARD_PREFIX, ''), 717 original_release_version=self.get_release_version(), 718 log_dir=self.job.resultdir, 719 force_update=force_update, 720 full_update=force_full_update, 721 force_original=force_original, 722 quick_provision=quick_provision) 723 724 725 def machine_install_by_devserver(self, update_url=None, force_update=False, 726 local_devserver=False, repair=False, 727 force_full_update=False): 728 """Ultiize devserver to install the DUT. 729 730 (TODO) crbugs.com/627269: The logic in this function has some overlap 731 with those in function machine_install. The merge will be done later, 732 not in the same CL. 733 734 @param update_url: The update_url or build for the host to update. 735 @param force_update: Force an update even if the version installed 736 is the same. Default:False 737 @param local_devserver: Used by test_that to allow people to 738 use their local devserver. Default: False 739 @param repair: Forces update to repair image. Implies force_update. 740 @param force_full_update: If True, do not attempt to run stateful 741 update, force a full reimage. If False, try stateful update 742 first when the dut is already installed with the same version. 743 @raises autoupdater.ChromiumOSError 744 745 @returns A tuple of (image_name, host_attributes). 746 image_name is the name of image installed, e.g., 747 veyron_jerry-release/R50-7871.0.0 748 host_attributes is a dictionary of (attribute, value), which 749 can be saved to afe_host_attributes table in database. This 750 method returns a dictionary with a single entry of 751 `job_repo_url`: repo_url, where repo_url is a devserver url to 752 autotest packages. 753 """ 754 if repair: 755 update_url = self._get_cros_repair_image_name() 756 force_update = True 757 758 if not update_url and not self._parser.options.image: 759 raise error.AutoservError( 760 'There is no update URL, nor a method to get one.') 761 762 if not update_url and self._parser.options.image: 763 update_url = self._parser.options.image 764 765 766 # Get build from parameter or AFE. 767 # If the build is not a URL, let devserver to stage it first. 768 # Otherwise, choose a devserver to trigger auto-update. 769 build = None 770 devserver = None 771 logging.debug('Resolving a devserver for auto-update') 772 previously_resolved = False 773 if update_url.startswith('http://'): 774 build = autoupdater.url_to_image_name(update_url) 775 previously_resolved = True 776 else: 777 build = update_url 778 devserver = dev_server.resolve(build, self.hostname) 779 server_name = devserver.hostname 780 781 monarch_fields = self._get_au_monarch_fields(devserver, build) 782 783 if previously_resolved: 784 # Make sure devserver for Auto-Update has staged the build. 785 if server_name not in update_url: 786 logging.debug('Resolved to devserver that does not match ' 787 'update_url. The previously resolved devserver ' 788 'must be unhealthy. Switching to use devserver %s,' 789 ' and re-staging.', 790 server_name) 791 logging.info('Staging build for AU: %s', update_url) 792 devserver.trigger_download(build, synchronous=False) 793 c = metrics.Counter( 794 'chromeos/autotest/provision/failover_download') 795 c.increment(fields=monarch_fields) 796 else: 797 logging.info('Staging build for AU: %s', update_url) 798 devserver.trigger_download(build, synchronous=False) 799 c = metrics.Counter('chromeos/autotest/provision/trigger_download') 800 c.increment(fields=monarch_fields) 801 802 # Report provision stats. 803 install_with_dev_counter = metrics.Counter( 804 'chromeos/autotest/provision/install_with_devserver') 805 install_with_dev_counter.increment(fields=monarch_fields) 806 logging.debug('Resolved devserver for auto-update: %s', devserver.url()) 807 808 # and other metrics from this function. 809 metrics.Counter('chromeos/autotest/provision/resolve' 810 ).increment(fields=monarch_fields) 811 812 force_original = self.get_chromeos_release_milestone() is None 813 814 build_re = CONFIG.get_config_value( 815 'CROS', 'quick_provision_build_regex', type=str, default='') 816 quick_provision = (len(build_re) != 0 and 817 re.match(build_re, build) is not None) 818 819 try: 820 devserver.auto_update( 821 self.hostname, build, 822 original_board=self.get_board().replace( 823 ds_constants.BOARD_PREFIX, ''), 824 original_release_version=self.get_release_version(), 825 log_dir=self.job.resultdir, 826 force_update=force_update, 827 full_update=force_full_update, 828 force_original=force_original, 829 quick_provision=quick_provision) 830 except dev_server.RetryableProvisionException: 831 # It indicates that last provision failed due to devserver load 832 # issue, so another devserver is resolved to kick off provision 833 # job once again and only once. 834 logging.debug('Provision failed due to devserver issue,' 835 'retry it with another devserver.') 836 837 # Check first whether this DUT is completely offline. If so, skip 838 # the following provision tries. 839 logging.debug('Checking whether host %s is online.', self.hostname) 840 if utils.ping(self.hostname, tries=1, deadline=1) == 0: 841 self._retry_auto_update_with_new_devserver( 842 build, devserver, force_update, force_full_update, 843 force_original, quick_provision) 844 else: 845 raise error.AutoservError( 846 'No answer to ping from %s' % self.hostname) 847 848 # The reason to resolve a new devserver in function machine_install 849 # is mostly because that the update_url there may has a strange format, 850 # and it's hard to parse the devserver url from it. 851 # Since we already resolve a devserver to trigger auto-update, the same 852 # devserver is used to form JOB_REPO_URL here. Verified in local test. 853 repo_url = tools.get_package_url(devserver.url(), build) 854 return build, {ds_constants.JOB_REPO_URL: repo_url} 855 856 857 def prepare_for_update(self): 858 """Prepares the DUT for an update. 859 860 Subclasses may override this to perform any special actions 861 required before updating. 862 """ 863 pass 864 865 866 def machine_install(self, update_url=None, force_update=False, 867 local_devserver=False, repair=False, 868 force_full_update=False): 869 """Install the DUT. 870 871 Use stateful update if the DUT is already running the same build. 872 Stateful update does not update kernel and tends to run much faster 873 than a full reimage. If the DUT is running a different build, or it 874 failed to do a stateful update, full update, including kernel update, 875 will be applied to the DUT. 876 877 Once a host enters machine_install its host attribute job_repo_url 878 (used for package install) will be removed and then updated. 879 880 @param update_url: The url to use for the update 881 pattern: http://$devserver:###/update/$build 882 If update_url is None and repair is True we will install the 883 stable image listed in afe_stable_versions table. If the table 884 is not setup, global_config value under CROS.stable_cros_version 885 will be used instead. 886 @param force_update: Force an update even if the version installed 887 is the same. Default:False 888 @param local_devserver: Used by test_that to allow people to 889 use their local devserver. Default: False 890 @param repair: Forces update to repair image. Implies force_update. 891 @param force_full_update: If True, do not attempt to run stateful 892 update, force a full reimage. If False, try stateful update 893 first when the dut is already installed with the same version. 894 @raises autoupdater.ChromiumOSError 895 896 @returns A tuple of (image_name, host_attributes). 897 image_name is the name of image installed, e.g., 898 veyron_jerry-release/R50-7871.0.0 899 host_attributes is a dictionary of (attribute, value), which 900 can be saved to afe_host_attributes table in database. This 901 method returns a dictionary with a single entry of 902 `job_repo_url`: repo_url, where repo_url is a devserver url to 903 autotest packages. 904 """ 905 devserver = None 906 if repair: 907 update_url, devserver = self._stage_image_for_update() 908 force_update = True 909 910 if not update_url and not self._parser.options.image: 911 raise error.AutoservError( 912 'There is no update URL, nor a method to get one.') 913 914 if not update_url and self._parser.options.image: 915 # This is the base case where we have no given update URL i.e. 916 # dynamic suites logic etc. This is the most flexible case where we 917 # can serve an update from any of our fleet of devservers. 918 requested_build = self._parser.options.image 919 if not requested_build.startswith('http://'): 920 logging.debug('Update will be staged for this installation') 921 update_url, devserver = self._stage_image_for_update( 922 requested_build) 923 else: 924 update_url = requested_build 925 926 logging.debug('Update URL is %s', update_url) 927 928 # Report provision stats. 929 server_name = dev_server.get_hostname(update_url) 930 (metrics.Counter('chromeos/autotest/provision/install') 931 .increment(fields={'devserver': server_name})) 932 933 # Create a file to indicate if provision fails. The file will be removed 934 # by stateful update or full install. 935 self.run('touch %s' % self.PROVISION_FAILED) 936 937 update_complete = False 938 updater = autoupdater.ChromiumOSUpdater( 939 update_url, host=self, local_devserver=local_devserver) 940 if not force_full_update: 941 try: 942 # If the DUT is already running the same build, try stateful 943 # update first as it's much quicker than a full re-image. 944 update_complete = self._try_stateful_update( 945 update_url, force_update, updater) 946 except Exception as e: 947 logging.exception(e) 948 949 inactive_kernel = None 950 if update_complete or (not force_update and updater.check_version()): 951 logging.info('Install complete without full update') 952 else: 953 logging.info('DUT requires full update.') 954 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) 955 self.prepare_for_update() 956 957 num_of_attempts = provision.FLAKY_DEVSERVER_ATTEMPTS 958 959 while num_of_attempts > 0: 960 num_of_attempts -= 1 961 try: 962 updater.run_update() 963 except Exception: 964 logging.warn('Autoupdate did not complete.') 965 # Do additional check for the devserver health. Ideally, 966 # the autoupdater.py could raise an exception when it 967 # detected network flake but that would require 968 # instrumenting the update engine and parsing it log. 969 if (num_of_attempts <= 0 or 970 devserver is None or 971 dev_server.ImageServer.devserver_healthy( 972 devserver.url())): 973 raise 974 975 logging.warn('Devserver looks unhealthy. Trying another') 976 update_url, devserver = self._stage_image_for_update( 977 requested_build) 978 logging.debug('New Update URL is %s', update_url) 979 updater = autoupdater.ChromiumOSUpdater( 980 update_url, host=self, 981 local_devserver=local_devserver) 982 else: 983 break 984 985 # Give it some time in case of IO issues. 986 time.sleep(10) 987 988 # Figure out active and inactive kernel. 989 active_kernel, inactive_kernel = updater.get_kernel_state() 990 991 # Ensure inactive kernel has higher priority than active. 992 if (updater.get_kernel_priority(inactive_kernel) 993 < updater.get_kernel_priority(active_kernel)): 994 raise autoupdater.ChromiumOSError( 995 'Update failed. The priority of the inactive kernel' 996 ' partition is less than that of the active kernel' 997 ' partition.') 998 999 # Updater has returned successfully; reboot the host. 1000 # 1001 # Regarding the 'crossystem' command: In some cases, the 1002 # TPM gets into a state such that it fails verification. 1003 # We don't know why. However, this call papers over the 1004 # problem by clearing the TPM during the reboot. 1005 # 1006 # We ignore failures from 'crossystem'. Although failure 1007 # here is unexpected, and could signal a bug, the point 1008 # of the exercise is to paper over problems; allowing 1009 # this to fail would defeat the purpose. 1010 self.run('crossystem clear_tpm_owner_request=1', 1011 ignore_status=True) 1012 self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True) 1013 1014 self._post_update_processing(updater, inactive_kernel) 1015 image_name = autoupdater.url_to_image_name(update_url) 1016 # update_url is different from devserver url needed to stage autotest 1017 # packages, therefore, resolve a new devserver url here. 1018 devserver_url = dev_server.ImageServer.resolve(image_name, 1019 self.hostname).url() 1020 repo_url = tools.get_package_url(devserver_url, image_name) 1021 return image_name, {ds_constants.JOB_REPO_URL: repo_url} 1022 1023 1024 def _clear_fw_version_labels(self, rw_only): 1025 """Clear firmware version labels from the machine. 1026 1027 @param rw_only: True to only clear fwrw_version; otherewise, clear 1028 both fwro_version and fwrw_version. 1029 """ 1030 labels = self._AFE.get_labels( 1031 name__startswith=provision.FW_RW_VERSION_PREFIX, 1032 host__hostname=self.hostname) 1033 if not rw_only: 1034 labels = labels + self._AFE.get_labels( 1035 name__startswith=provision.FW_RO_VERSION_PREFIX, 1036 host__hostname=self.hostname) 1037 for label in labels: 1038 label.remove_hosts(hosts=[self.hostname]) 1039 1040 1041 def _add_fw_version_label(self, build, rw_only): 1042 """Add firmware version label to the machine. 1043 1044 @param build: Build of firmware. 1045 @param rw_only: True to only add fwrw_version; otherwise, add both 1046 fwro_version and fwrw_version. 1047 1048 """ 1049 fw_label = provision.fwrw_version_to_label(build) 1050 self._AFE.run('label_add_hosts', id=fw_label, hosts=[self.hostname]) 1051 if not rw_only: 1052 fw_label = provision.fwro_version_to_label(build) 1053 self._AFE.run('label_add_hosts', id=fw_label, hosts=[self.hostname]) 1054 1055 1056 def firmware_install(self, build=None, rw_only=False): 1057 """Install firmware to the DUT. 1058 1059 Use stateful update if the DUT is already running the same build. 1060 Stateful update does not update kernel and tends to run much faster 1061 than a full reimage. If the DUT is running a different build, or it 1062 failed to do a stateful update, full update, including kernel update, 1063 will be applied to the DUT. 1064 1065 Once a host enters firmware_install its fw[ro|rw]_version label will 1066 be removed. After the firmware is updated successfully, a new 1067 fw[ro|rw]_version label will be added to the host. 1068 1069 @param build: The build version to which we want to provision the 1070 firmware of the machine, 1071 e.g. 'link-firmware/R22-2695.1.144'. 1072 @param rw_only: True to only install firmware to its RW portions. Keep 1073 the RO portions unchanged. 1074 1075 TODO(dshi): After bug 381718 is fixed, update here with corresponding 1076 exceptions that could be raised. 1077 1078 """ 1079 if not self.servo: 1080 raise error.TestError('Host %s does not have servo.' % 1081 self.hostname) 1082 1083 # Get the DUT board name from servod. 1084 board = self.servo.get_board() 1085 1086 # If build is not set, try to install firmware from stable CrOS. 1087 if not build: 1088 build = afe_utils.get_stable_faft_version(board) 1089 if not build: 1090 raise error.TestError( 1091 'Failed to find stable firmware build for %s.', 1092 self.hostname) 1093 logging.info('Will install firmware from build %s.', build) 1094 1095 config = FAFTConfig(board) 1096 if config.use_u_boot: 1097 ap_image = 'image-%s.bin' % board 1098 else: # Depthcharge platform 1099 ap_image = 'image.bin' 1100 ec_image = 'ec.bin' 1101 ds = dev_server.ImageServer.resolve(build, self.hostname) 1102 ds.stage_artifacts(build, ['firmware']) 1103 1104 tmpd = autotemp.tempdir(unique_id='fwimage') 1105 try: 1106 fwurl = self._FW_IMAGE_URL_PATTERN % (ds.url(), build) 1107 local_tarball = os.path.join(tmpd.name, os.path.basename(fwurl)) 1108 ds.download_file(fwurl, local_tarball) 1109 1110 self._clear_fw_version_labels(rw_only) 1111 if config.chrome_ec: 1112 logging.info('Will re-program EC %snow', 'RW ' if rw_only else '') 1113 server_utils.system('tar xf %s -C %s %s' % 1114 (local_tarball, tmpd.name, ec_image), 1115 timeout=60) 1116 self.servo.program_ec(os.path.join(tmpd.name, ec_image), rw_only) 1117 else: 1118 logging.info('Not a Chrome EC, ignore re-programing it') 1119 logging.info('Will re-program BIOS %snow', 'RW ' if rw_only else '') 1120 server_utils.system('tar xf %s -C %s %s' % 1121 (local_tarball, tmpd.name, ap_image), 1122 timeout=60) 1123 self.servo.program_bios(os.path.join(tmpd.name, ap_image), rw_only) 1124 self.servo.get_power_state_controller().reset() 1125 time.sleep(self.servo.BOOT_DELAY) 1126 if utils.host_is_in_lab_zone(self.hostname): 1127 self._add_fw_version_label(build, rw_only) 1128 finally: 1129 tmpd.clean() 1130 1131 1132 def program_base_ec(self, image_path): 1133 """Program Base EC on DUT with the given image. 1134 1135 @param image_path: a string, file name of the EC image to program 1136 on the DUT. 1137 1138 """ 1139 dest_path = os.path.join('/tmp', os.path.basename(image_path)) 1140 self.send_file(image_path, dest_path) 1141 programmer = firmware_programmer.ProgrammerDfu(self.servo, self) 1142 programmer.program_ec(dest_path) 1143 1144 1145 def show_update_engine_log(self): 1146 """Output update engine log.""" 1147 logging.debug('Dumping %s', client_constants.UPDATE_ENGINE_LOG) 1148 self.run('cat %s' % client_constants.UPDATE_ENGINE_LOG) 1149 1150 1151 def servo_install(self, image_url=None, usb_boot_timeout=USB_BOOT_TIMEOUT, 1152 install_timeout=INSTALL_TIMEOUT): 1153 """ 1154 Re-install the OS on the DUT by: 1155 1) installing a test image on a USB storage device attached to the Servo 1156 board, 1157 2) booting that image in recovery mode, and then 1158 3) installing the image with chromeos-install. 1159 1160 @param image_url: If specified use as the url to install on the DUT. 1161 otherwise boot the currently staged image on the USB stick. 1162 @param usb_boot_timeout: The usb_boot_timeout to use during reimage. 1163 Factory images need a longer usb_boot_timeout than regular 1164 cros images. 1165 @param install_timeout: The timeout to use when installing the chromeos 1166 image. Factory images need a longer install_timeout. 1167 1168 @raises AutoservError if the image fails to boot. 1169 1170 """ 1171 logging.info('Downloading image to USB, then booting from it. Usb boot ' 1172 'timeout = %s', usb_boot_timeout) 1173 with metrics.SecondsTimer( 1174 'chromeos/autotest/provision/servo_install/boot_duration'): 1175 self.servo.install_recovery_image(image_url) 1176 if not self.wait_up(timeout=usb_boot_timeout): 1177 raise hosts.AutoservRepairError( 1178 'DUT failed to boot from USB after %d seconds' % 1179 usb_boot_timeout) 1180 1181 # The new chromeos-tpm-recovery has been merged since R44-7073.0.0. 1182 # In old CrOS images, this command fails. Skip the error. 1183 logging.info('Resetting the TPM status') 1184 try: 1185 self.run('chromeos-tpm-recovery') 1186 except error.AutoservRunError: 1187 logging.warn('chromeos-tpm-recovery is too old.') 1188 1189 1190 with metrics.SecondsTimer( 1191 'chromeos/autotest/provision/servo_install/install_duration'): 1192 logging.info('Installing image through chromeos-install.') 1193 self.run('chromeos-install --yes', timeout=install_timeout) 1194 self.halt() 1195 1196 logging.info('Power cycling DUT through servo.') 1197 self.servo.get_power_state_controller().power_off() 1198 self.servo.switch_usbkey('off') 1199 # N.B. The Servo API requires that we use power_on() here 1200 # for two reasons: 1201 # 1) After turning on a DUT in recovery mode, you must turn 1202 # it off and then on with power_on() once more to 1203 # disable recovery mode (this is a Parrot specific 1204 # requirement). 1205 # 2) After power_off(), the only way to turn on is with 1206 # power_on() (this is a Storm specific requirement). 1207 self.servo.get_power_state_controller().power_on() 1208 1209 logging.info('Waiting for DUT to come back up.') 1210 if not self.wait_up(timeout=self.BOOT_TIMEOUT): 1211 raise error.AutoservError('DUT failed to reboot installed ' 1212 'test image after %d seconds' % 1213 self.BOOT_TIMEOUT) 1214 1215 1216 def repair_servo(self): 1217 """ 1218 Confirm that servo is initialized and verified. 1219 1220 If the servo object is missing, attempt to repair the servo 1221 host. Repair failures are passed back to the caller. 1222 1223 @raise AutoservError: If there is no servo host for this CrOS 1224 host. 1225 """ 1226 if self.servo: 1227 return 1228 if not self._servo_host: 1229 raise error.AutoservError('No servo host for %s.' % 1230 self.hostname) 1231 self._servo_host.repair() 1232 self.servo = self._servo_host.get_servo() 1233 1234 1235 def repair(self): 1236 """Attempt to get the DUT to pass `self.verify()`. 1237 1238 This overrides the base class function for repair; it does 1239 not call back to the parent class, but instead relies on 1240 `self._repair_strategy` to coordinate the verification and 1241 repair steps needed to get the DUT working. 1242 """ 1243 self._repair_strategy.repair(self) 1244 # Sometimes, hosts with certain ethernet dongles get stuck in a 1245 # bad network state where they're reachable from this code, but 1246 # not from the devservers during provisioning. Rebooting the 1247 # DUT fixes it. 1248 # 1249 # TODO(jrbarnette): Ideally, we'd get rid of the problem 1250 # dongles, and drop this code. Failing that, we could be smart 1251 # enough not to reboot if repair rebooted the DUT (e.g. by 1252 # looking at DUT uptime after repair completes). 1253 1254 self.reboot() 1255 1256 1257 def close(self): 1258 """Close connection.""" 1259 super(CrosHost, self).close() 1260 if self._chameleon_host: 1261 self._chameleon_host.close() 1262 1263 if self._servo_host: 1264 self._servo_host.close() 1265 1266 1267 def get_power_supply_info(self): 1268 """Get the output of power_supply_info. 1269 1270 power_supply_info outputs the info of each power supply, e.g., 1271 Device: Line Power 1272 online: no 1273 type: Mains 1274 voltage (V): 0 1275 current (A): 0 1276 Device: Battery 1277 state: Discharging 1278 percentage: 95.9276 1279 technology: Li-ion 1280 1281 Above output shows two devices, Line Power and Battery, with details of 1282 each device listed. This function parses the output into a dictionary, 1283 with key being the device name, and value being a dictionary of details 1284 of the device info. 1285 1286 @return: The dictionary of power_supply_info, e.g., 1287 {'Line Power': {'online': 'yes', 'type': 'main'}, 1288 'Battery': {'vendor': 'xyz', 'percentage': '100'}} 1289 @raise error.AutoservRunError if power_supply_info tool is not found in 1290 the DUT. Caller should handle this error to avoid false failure 1291 on verification. 1292 """ 1293 result = self.run('power_supply_info').stdout.strip() 1294 info = {} 1295 device_name = None 1296 device_info = {} 1297 for line in result.split('\n'): 1298 pair = [v.strip() for v in line.split(':')] 1299 if len(pair) != 2: 1300 continue 1301 if pair[0] == 'Device': 1302 if device_name: 1303 info[device_name] = device_info 1304 device_name = pair[1] 1305 device_info = {} 1306 else: 1307 device_info[pair[0]] = pair[1] 1308 if device_name and not device_name in info: 1309 info[device_name] = device_info 1310 return info 1311 1312 1313 def get_battery_percentage(self): 1314 """Get the battery percentage. 1315 1316 @return: The percentage of battery level, value range from 0-100. Return 1317 None if the battery info cannot be retrieved. 1318 """ 1319 try: 1320 info = self.get_power_supply_info() 1321 logging.info(info) 1322 return float(info['Battery']['percentage']) 1323 except (KeyError, ValueError, error.AutoservRunError): 1324 return None 1325 1326 1327 def is_ac_connected(self): 1328 """Check if the dut has power adapter connected and charging. 1329 1330 @return: True if power adapter is connected and charging. 1331 """ 1332 try: 1333 info = self.get_power_supply_info() 1334 return info['Line Power']['online'] == 'yes' 1335 except (KeyError, error.AutoservRunError): 1336 return None 1337 1338 1339 def _cleanup_poweron(self): 1340 """Special cleanup method to make sure hosts always get power back.""" 1341 afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10) 1342 hosts = afe.get_hosts(hostname=self.hostname) 1343 if not hosts or not (self._RPM_OUTLET_CHANGED in 1344 hosts[0].attributes): 1345 return 1346 logging.debug('This host has recently interacted with the RPM' 1347 ' Infrastructure. Ensuring power is on.') 1348 try: 1349 self.power_on() 1350 afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None, 1351 hostname=self.hostname) 1352 except rpm_client.RemotePowerException: 1353 logging.error('Failed to turn Power On for this host after ' 1354 'cleanup through the RPM Infrastructure.') 1355 1356 battery_percentage = self.get_battery_percentage() 1357 if battery_percentage and battery_percentage < 50: 1358 raise 1359 elif self.is_ac_connected(): 1360 logging.info('The device has power adapter connected and ' 1361 'charging. No need to try to turn RPM on ' 1362 'again.') 1363 afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None, 1364 hostname=self.hostname) 1365 logging.info('Battery level is now at %s%%. The device may ' 1366 'still have enough power to run test, so no ' 1367 'exception will be raised.', battery_percentage) 1368 1369 1370 def _is_factory_image(self): 1371 """Checks if the image on the DUT is a factory image. 1372 1373 @return: True if the image on the DUT is a factory image. 1374 False otherwise. 1375 """ 1376 result = self.run('[ -f /root/.factory_test ]', ignore_status=True) 1377 return result.exit_status == 0 1378 1379 1380 def _restart_ui(self): 1381 """Restart the Chrome UI. 1382 1383 @raises: FactoryImageCheckerException for factory images, since 1384 we cannot attempt to restart ui on them. 1385 error.AutoservRunError for any other type of error that 1386 occurs while restarting ui. 1387 """ 1388 if self._is_factory_image(): 1389 raise FactoryImageCheckerException('Cannot restart ui on factory ' 1390 'images') 1391 1392 # TODO(jrbarnette): The command to stop/start the ui job 1393 # should live inside cros_ui, too. However that would seem 1394 # to imply interface changes to the existing start()/restart() 1395 # functions, which is a bridge too far (for now). 1396 prompt = cros_ui.get_chrome_session_ident(self) 1397 self.run('stop ui; start ui') 1398 cros_ui.wait_for_chrome_ready(prompt, self) 1399 1400 1401 def _get_lsb_release_content(self): 1402 """Return the content of lsb-release file of host.""" 1403 return self.run( 1404 'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip() 1405 1406 1407 def get_release_version(self): 1408 """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release. 1409 1410 @returns The version string in lsb-release, under attribute 1411 CHROMEOS_RELEASE_VERSION. 1412 """ 1413 return lsbrelease_utils.get_chromeos_release_version( 1414 lsb_release_content=self._get_lsb_release_content()) 1415 1416 1417 def get_release_builder_path(self): 1418 """Get the value of CHROMEOS_RELEASE_BUILDER_PATH from lsb-release. 1419 1420 @returns The version string in lsb-release, under attribute 1421 CHROMEOS_RELEASE_BUILDER_PATH. 1422 """ 1423 return lsbrelease_utils.get_chromeos_release_builder_path( 1424 lsb_release_content=self._get_lsb_release_content()) 1425 1426 1427 def get_chromeos_release_milestone(self): 1428 """Get the value of attribute CHROMEOS_RELEASE_BUILD_TYPE 1429 from lsb-release. 1430 1431 @returns The version string in lsb-release, under attribute 1432 CHROMEOS_RELEASE_BUILD_TYPE. 1433 """ 1434 return lsbrelease_utils.get_chromeos_release_milestone( 1435 lsb_release_content=self._get_lsb_release_content()) 1436 1437 1438 def verify_cros_version_label(self): 1439 """ Make sure host's cros-version label match the actual image in dut. 1440 1441 Remove any cros-version: label that doesn't match that installed in 1442 the dut. 1443 1444 @param raise_error: Set to True to raise exception if any mismatch found 1445 1446 @raise error.AutoservError: If any mismatch between cros-version label 1447 and the build installed in dut is found. 1448 """ 1449 labels = self._AFE.get_labels( 1450 name__startswith=ds_constants.VERSION_PREFIX, 1451 host__hostname=self.hostname) 1452 mismatch_found = False 1453 if labels: 1454 # Ask the DUT for its canonical image name. This will be in 1455 # a form like this: kevin-release/R66-10405.0.0 1456 release_builder_path = self.get_release_builder_path() 1457 host_list = [self.hostname] 1458 for label in labels: 1459 # Remove any cros-version label that does not match 1460 # the DUT's installed image. 1461 # 1462 # TODO(jrbarnette): Tests sent to the `arc-presubmit` 1463 # pool install images matching the format above, but 1464 # then apply a label with `-cheetsth` appended. Probably, 1465 # it's wrong for ARC presubmit testing to make that change, 1466 # but until it's fixed, this code specifically excuses that 1467 # behavior. 1468 build_version = label.name[len(ds_constants.VERSION_PREFIX):] 1469 if build_version.endswith('-cheetsth'): 1470 build_version = build_version[:-len('-cheetsth')] 1471 if build_version != release_builder_path: 1472 logging.warn( 1473 'cros-version label "%s" does not match ' 1474 'release_builder_path %s. Removing the label.', 1475 label.name, release_builder_path) 1476 label.remove_hosts(hosts=host_list) 1477 mismatch_found = True 1478 if mismatch_found: 1479 raise error.AutoservError('The host has wrong cros-version label.') 1480 1481 1482 def cleanup_services(self): 1483 """Reinitializes the device for cleanup. 1484 1485 Subclasses may override this to customize the cleanup method. 1486 1487 To indicate failure of the reset, the implementation may raise 1488 any of: 1489 error.AutoservRunError 1490 error.AutotestRunError 1491 FactoryImageCheckerException 1492 1493 @raises error.AutoservRunError 1494 @raises error.AutotestRunError 1495 @raises error.FactoryImageCheckerException 1496 """ 1497 self._restart_ui() 1498 1499 1500 def cleanup(self): 1501 """Cleanup state on device.""" 1502 self.run('rm -f %s' % client_constants.CLEANUP_LOGS_PAUSED_FILE) 1503 try: 1504 self.cleanup_services() 1505 except (error.AutotestRunError, error.AutoservRunError, 1506 FactoryImageCheckerException): 1507 logging.warning('Unable to restart ui, rebooting device.') 1508 # Since restarting the UI fails fall back to normal Autotest 1509 # cleanup routines, i.e. reboot the machine. 1510 super(CrosHost, self).cleanup() 1511 # Check if the rpm outlet was manipulated. 1512 if self.has_power(): 1513 self._cleanup_poweron() 1514 self.verify_cros_version_label() 1515 1516 1517 def reboot(self, **dargs): 1518 """ 1519 This function reboots the site host. The more generic 1520 RemoteHost.reboot() performs sync and sleeps for 5 1521 seconds. This is not necessary for Chrome OS devices as the 1522 sync should be finished in a short time during the reboot 1523 command. 1524 """ 1525 if 'reboot_cmd' not in dargs: 1526 reboot_timeout = dargs.get('reboot_timeout', 10) 1527 dargs['reboot_cmd'] = ('sleep 1; ' 1528 'reboot & sleep %d; ' 1529 'reboot -f' % reboot_timeout) 1530 # Enable fastsync to avoid running extra sync commands before reboot. 1531 if 'fastsync' not in dargs: 1532 dargs['fastsync'] = True 1533 1534 # For purposes of logging reboot times: 1535 # Get the board name i.e. 'daisy_spring' 1536 board_fullname = self.get_board() 1537 1538 # Strip the prefix and add it to dargs. 1539 dargs['board'] = board_fullname[board_fullname.find(':')+1:] 1540 # Record who called us 1541 orig = sys._getframe(1).f_code 1542 metric_fields = {'board' : dargs['board'], 1543 'dut_host_name' : self.hostname, 1544 'success' : True} 1545 metric_debug_fields = {'board' : dargs['board'], 1546 'caller' : "%s:%s" % (orig.co_filename, orig.co_name), 1547 'success' : True, 1548 'error' : ''} 1549 1550 t0 = time.time() 1551 try: 1552 super(CrosHost, self).reboot(**dargs) 1553 except Exception as e: 1554 metric_fields['success'] = False 1555 metric_debug_fields['success'] = False 1556 metric_debug_fields['error'] = type(e).__name__ 1557 raise 1558 finally: 1559 duration = int(time.time() - t0) 1560 metrics.Counter( 1561 'chromeos/autotest/autoserv/reboot_count').increment( 1562 fields=metric_fields) 1563 metrics.Counter( 1564 'chromeos/autotest/autoserv/reboot_debug').increment( 1565 fields=metric_debug_fields) 1566 metrics.SecondsDistribution( 1567 'chromeos/autotest/autoserv/reboot_duration').add( 1568 duration, fields=metric_fields) 1569 1570 1571 def suspend(self, **dargs): 1572 """ 1573 This function suspends the site host. 1574 """ 1575 suspend_time = dargs.get('suspend_time', 60) 1576 dargs['timeout'] = suspend_time 1577 if 'suspend_cmd' not in dargs: 1578 dargs['suspend_cmd'] = ' && '.join([ 1579 'echo 0 > /sys/class/rtc/rtc0/wakealarm', 1580 'echo +%d > /sys/class/rtc/rtc0/wakealarm' % suspend_time, 1581 'powerd_dbus_suspend --delay=0']) 1582 super(CrosHost, self).suspend(**dargs) 1583 1584 1585 def upstart_status(self, service_name): 1586 """Check the status of an upstart init script. 1587 1588 @param service_name: Service to look up. 1589 1590 @returns True if the service is running, False otherwise. 1591 """ 1592 return 'start/running' in self.run('status %s' % service_name, 1593 ignore_status=True).stdout 1594 1595 1596 def verify_software(self): 1597 """Verify working software on a Chrome OS system. 1598 1599 Tests for the following conditions: 1600 1. All conditions tested by the parent version of this 1601 function. 1602 2. Sufficient space in /mnt/stateful_partition. 1603 3. Sufficient space in /mnt/stateful_partition/encrypted. 1604 4. update_engine answers a simple status request over DBus. 1605 1606 """ 1607 super(CrosHost, self).verify_software() 1608 default_kilo_inodes_required = CONFIG.get_config_value( 1609 'SERVER', 'kilo_inodes_required', type=int, default=100) 1610 board = self.get_board().replace(ds_constants.BOARD_PREFIX, '') 1611 kilo_inodes_required = CONFIG.get_config_value( 1612 'SERVER', 'kilo_inodes_required_%s' % board, 1613 type=int, default=default_kilo_inodes_required) 1614 self.check_inodes('/mnt/stateful_partition', kilo_inodes_required) 1615 self.check_diskspace( 1616 '/mnt/stateful_partition', 1617 CONFIG.get_config_value( 1618 'SERVER', 'gb_diskspace_required', type=float, 1619 default=20.0)) 1620 encrypted_stateful_path = '/mnt/stateful_partition/encrypted' 1621 # Not all targets build with encrypted stateful support. 1622 if self.path_exists(encrypted_stateful_path): 1623 self.check_diskspace( 1624 encrypted_stateful_path, 1625 CONFIG.get_config_value( 1626 'SERVER', 'gb_encrypted_diskspace_required', type=float, 1627 default=0.1)) 1628 1629 self.wait_for_system_services() 1630 1631 # Factory images don't run update engine, 1632 # goofy controls dbus on these DUTs. 1633 if not self._is_factory_image(): 1634 self.run('update_engine_client --status') 1635 1636 self.verify_cros_version_label() 1637 1638 1639 @retry.retry(error.AutoservError, timeout_min=5, delay_sec=10) 1640 def wait_for_system_services(self): 1641 """Waits for system-services to be running. 1642 1643 Sometimes, update_engine will take a while to update firmware, so we 1644 should give this some time to finish. See crbug.com/765686#c38 for 1645 details. 1646 """ 1647 if not self.upstart_status('system-services'): 1648 raise error.AutoservError('Chrome failed to reach login. ' 1649 'System services not running.') 1650 1651 1652 def verify(self): 1653 """Verify Chrome OS system is in good state.""" 1654 self._repair_strategy.verify(self) 1655 1656 1657 def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None, 1658 connect_timeout=None, alive_interval=None, 1659 alive_count_max=None, connection_attempts=None): 1660 """Override default make_ssh_command to use options tuned for Chrome OS. 1661 1662 Tuning changes: 1663 - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH 1664 connection failure. Consistency with remote_access.sh. 1665 1666 - ServerAliveInterval=900; which causes SSH to ping connection every 1667 900 seconds. In conjunction with ServerAliveCountMax ensures 1668 that if the connection dies, Autotest will bail out. 1669 Originally tried 60 secs, but saw frequent job ABORTS where 1670 the test completed successfully. Later increased from 180 seconds to 1671 900 seconds to account for tests where the DUT is suspended for 1672 longer periods of time. 1673 1674 - ServerAliveCountMax=3; consistency with remote_access.sh. 1675 1676 - ConnectAttempts=4; reduce flakiness in connection errors; 1677 consistency with remote_access.sh. 1678 1679 - UserKnownHostsFile=/dev/null; we don't care about the keys. 1680 Host keys change with every new installation, don't waste 1681 memory/space saving them. 1682 1683 - SSH protocol forced to 2; needed for ServerAliveInterval. 1684 1685 @param user User name to use for the ssh connection. 1686 @param port Port on the target host to use for ssh connection. 1687 @param opts Additional options to the ssh command. 1688 @param hosts_file Ignored. 1689 @param connect_timeout Ignored. 1690 @param alive_interval Ignored. 1691 @param alive_count_max Ignored. 1692 @param connection_attempts Ignored. 1693 """ 1694 options = ' '.join([opts, '-o Protocol=2']) 1695 return super(CrosHost, self).make_ssh_command( 1696 user=user, port=port, opts=options, hosts_file='/dev/null', 1697 connect_timeout=30, alive_interval=900, alive_count_max=3, 1698 connection_attempts=4) 1699 1700 1701 def syslog(self, message, tag='autotest'): 1702 """Logs a message to syslog on host. 1703 1704 @param message String message to log into syslog 1705 @param tag String tag prefix for syslog 1706 1707 """ 1708 self.run('logger -t "%s" "%s"' % (tag, message)) 1709 1710 1711 def _ping_check_status(self, status): 1712 """Ping the host once, and return whether it has a given status. 1713 1714 @param status Check the ping status against this value. 1715 @return True iff `status` and the result of ping are the same 1716 (i.e. both True or both False). 1717 1718 """ 1719 ping_val = utils.ping(self.hostname, tries=1, deadline=1) 1720 return not (status ^ (ping_val == 0)) 1721 1722 def _ping_wait_for_status(self, status, timeout): 1723 """Wait for the host to have a given status (UP or DOWN). 1724 1725 Status is checked by polling. Polling will not last longer 1726 than the number of seconds in `timeout`. The polling 1727 interval will be long enough that only approximately 1728 _PING_WAIT_COUNT polling cycles will be executed, subject 1729 to a maximum interval of about one minute. 1730 1731 @param status Waiting will stop immediately if `ping` of the 1732 host returns this status. 1733 @param timeout Poll for at most this many seconds. 1734 @return True iff the host status from `ping` matched the 1735 requested status at the time of return. 1736 1737 """ 1738 # _ping_check_status() takes about 1 second, hence the 1739 # "- 1" in the formula below. 1740 # FIXME: if the ping command errors then _ping_check_status() 1741 # returns instantly. If timeout is also smaller than twice 1742 # _PING_WAIT_COUNT then the while loop below forks many 1743 # thousands of ping commands (see /tmp/test_that_results_XXXXX/ 1744 # /results-1-logging_YYY.ZZZ/debug/autoserv.DEBUG) and hogs one 1745 # CPU core for 60 seconds. 1746 poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1 1747 end_time = time.time() + timeout 1748 while time.time() <= end_time: 1749 if self._ping_check_status(status): 1750 return True 1751 if poll_interval > 0: 1752 time.sleep(poll_interval) 1753 1754 # The last thing we did was sleep(poll_interval), so it may 1755 # have been too long since the last `ping`. Check one more 1756 # time, just to be sure. 1757 return self._ping_check_status(status) 1758 1759 def ping_wait_up(self, timeout): 1760 """Wait for the host to respond to `ping`. 1761 1762 N.B. This method is not a reliable substitute for 1763 `wait_up()`, because a host that responds to ping will not 1764 necessarily respond to ssh. This method should only be used 1765 if the target DUT can be considered functional even if it 1766 can't be reached via ssh. 1767 1768 @param timeout Minimum time to allow before declaring the 1769 host to be non-responsive. 1770 @return True iff the host answered to ping before the timeout. 1771 1772 """ 1773 return self._ping_wait_for_status(self._PING_STATUS_UP, timeout) 1774 1775 def ping_wait_down(self, timeout): 1776 """Wait until the host no longer responds to `ping`. 1777 1778 This function can be used as a slightly faster version of 1779 `wait_down()`, by avoiding potentially long ssh timeouts. 1780 1781 @param timeout Minimum time to allow for the host to become 1782 non-responsive. 1783 @return True iff the host quit answering ping before the 1784 timeout. 1785 1786 """ 1787 return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout) 1788 1789 def test_wait_for_sleep(self, sleep_timeout=None): 1790 """Wait for the client to enter low-power sleep mode. 1791 1792 The test for "is asleep" can't distinguish a system that is 1793 powered off; to confirm that the unit was asleep, it is 1794 necessary to force resume, and then call 1795 `test_wait_for_resume()`. 1796 1797 This function is expected to be called from a test as part 1798 of a sequence like the following: 1799 1800 ~~~~~~~~ 1801 boot_id = host.get_boot_id() 1802 # trigger sleep on the host 1803 host.test_wait_for_sleep() 1804 # trigger resume on the host 1805 host.test_wait_for_resume(boot_id) 1806 ~~~~~~~~ 1807 1808 @param sleep_timeout time limit in seconds to allow the host sleep. 1809 1810 @exception TestFail The host did not go to sleep within 1811 the allowed time. 1812 """ 1813 if sleep_timeout is None: 1814 sleep_timeout = self.SLEEP_TIMEOUT 1815 1816 if not self.ping_wait_down(timeout=sleep_timeout): 1817 raise error.TestFail( 1818 'client failed to sleep after %d seconds' % sleep_timeout) 1819 1820 1821 def test_wait_for_resume(self, old_boot_id, resume_timeout=None): 1822 """Wait for the client to resume from low-power sleep mode. 1823 1824 The `old_boot_id` parameter should be the value from 1825 `get_boot_id()` obtained prior to entering sleep mode. A 1826 `TestFail` exception is raised if the boot id changes. 1827 1828 See @ref test_wait_for_sleep for more on this function's 1829 usage. 1830 1831 @param old_boot_id A boot id value obtained before the 1832 target host went to sleep. 1833 @param resume_timeout time limit in seconds to allow the host up. 1834 1835 @exception TestFail The host did not respond within the 1836 allowed time. 1837 @exception TestFail The host responded, but the boot id test 1838 indicated a reboot rather than a sleep 1839 cycle. 1840 """ 1841 if resume_timeout is None: 1842 resume_timeout = self.RESUME_TIMEOUT 1843 1844 if not self.wait_up(timeout=resume_timeout): 1845 raise error.TestFail( 1846 'client failed to resume from sleep after %d seconds' % 1847 resume_timeout) 1848 else: 1849 new_boot_id = self.get_boot_id() 1850 if new_boot_id != old_boot_id: 1851 logging.error('client rebooted (old boot %s, new boot %s)', 1852 old_boot_id, new_boot_id) 1853 raise error.TestFail( 1854 'client rebooted, but sleep was expected') 1855 1856 1857 def test_wait_for_shutdown(self, shutdown_timeout=None): 1858 """Wait for the client to shut down. 1859 1860 The test for "has shut down" can't distinguish a system that 1861 is merely asleep; to confirm that the unit was down, it is 1862 necessary to force boot, and then call test_wait_for_boot(). 1863 1864 This function is expected to be called from a test as part 1865 of a sequence like the following: 1866 1867 ~~~~~~~~ 1868 boot_id = host.get_boot_id() 1869 # trigger shutdown on the host 1870 host.test_wait_for_shutdown() 1871 # trigger boot on the host 1872 host.test_wait_for_boot(boot_id) 1873 ~~~~~~~~ 1874 1875 @param shutdown_timeout time limit in seconds to allow the host down. 1876 @exception TestFail The host did not shut down within the 1877 allowed time. 1878 """ 1879 if shutdown_timeout is None: 1880 shutdown_timeout = self.SHUTDOWN_TIMEOUT 1881 1882 if not self.ping_wait_down(timeout=shutdown_timeout): 1883 raise error.TestFail( 1884 'client failed to shut down after %d seconds' % 1885 shutdown_timeout) 1886 1887 1888 def test_wait_for_boot(self, old_boot_id=None): 1889 """Wait for the client to boot from cold power. 1890 1891 The `old_boot_id` parameter should be the value from 1892 `get_boot_id()` obtained prior to shutting down. A 1893 `TestFail` exception is raised if the boot id does not 1894 change. The boot id test is omitted if `old_boot_id` is not 1895 specified. 1896 1897 See @ref test_wait_for_shutdown for more on this function's 1898 usage. 1899 1900 @param old_boot_id A boot id value obtained before the 1901 shut down. 1902 1903 @exception TestFail The host did not respond within the 1904 allowed time. 1905 @exception TestFail The host responded, but the boot id test 1906 indicated that there was no reboot. 1907 """ 1908 if not self.wait_up(timeout=self.REBOOT_TIMEOUT): 1909 raise error.TestFail( 1910 'client failed to reboot after %d seconds' % 1911 self.REBOOT_TIMEOUT) 1912 elif old_boot_id: 1913 if self.get_boot_id() == old_boot_id: 1914 logging.error('client not rebooted (boot %s)', 1915 old_boot_id) 1916 raise error.TestFail( 1917 'client is back up, but did not reboot') 1918 1919 1920 @staticmethod 1921 def check_for_rpm_support(hostname): 1922 """For a given hostname, return whether or not it is powered by an RPM. 1923 1924 @param hostname: hostname to check for rpm support. 1925 1926 @return None if this host does not follows the defined naming format 1927 for RPM powered DUT's in the lab. If it does follow the format, 1928 it returns a regular expression MatchObject instead. 1929 """ 1930 return re.match(CrosHost._RPM_HOSTNAME_REGEX, hostname) 1931 1932 1933 def has_power(self): 1934 """For this host, return whether or not it is powered by an RPM. 1935 1936 @return True if this host is in the CROS lab and follows the defined 1937 naming format. 1938 """ 1939 return CrosHost.check_for_rpm_support(self.hostname) 1940 1941 1942 def _set_power(self, state, power_method): 1943 """Sets the power to the host via RPM, Servo or manual. 1944 1945 @param state Specifies which power state to set to DUT 1946 @param power_method Specifies which method of power control to 1947 use. By default "RPM" will be used. Valid values 1948 are the strings "RPM", "manual", "servoj10". 1949 1950 """ 1951 ACCEPTABLE_STATES = ['ON', 'OFF'] 1952 1953 if state.upper() not in ACCEPTABLE_STATES: 1954 raise error.TestError('State must be one of: %s.' 1955 % (ACCEPTABLE_STATES,)) 1956 1957 if power_method == self.POWER_CONTROL_SERVO: 1958 logging.info('Setting servo port J10 to %s', state) 1959 self.servo.set('prtctl3_pwren', state.lower()) 1960 time.sleep(self._USB_POWER_TIMEOUT) 1961 elif power_method == self.POWER_CONTROL_MANUAL: 1962 logging.info('You have %d seconds to set the AC power to %s.', 1963 self._POWER_CYCLE_TIMEOUT, state) 1964 time.sleep(self._POWER_CYCLE_TIMEOUT) 1965 else: 1966 if not self.has_power(): 1967 raise error.TestFail('DUT does not have RPM connected.') 1968 afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10) 1969 afe.set_host_attribute(self._RPM_OUTLET_CHANGED, True, 1970 hostname=self.hostname) 1971 rpm_client.set_power(self.hostname, state.upper(), timeout_mins=5) 1972 1973 1974 def power_off(self, power_method=POWER_CONTROL_RPM): 1975 """Turn off power to this host via RPM, Servo or manual. 1976 1977 @param power_method Specifies which method of power control to 1978 use. By default "RPM" will be used. Valid values 1979 are the strings "RPM", "manual", "servoj10". 1980 1981 """ 1982 self._set_power('OFF', power_method) 1983 1984 1985 def power_on(self, power_method=POWER_CONTROL_RPM): 1986 """Turn on power to this host via RPM, Servo or manual. 1987 1988 @param power_method Specifies which method of power control to 1989 use. By default "RPM" will be used. Valid values 1990 are the strings "RPM", "manual", "servoj10". 1991 1992 """ 1993 self._set_power('ON', power_method) 1994 1995 1996 def power_cycle(self, power_method=POWER_CONTROL_RPM): 1997 """Cycle power to this host by turning it OFF, then ON. 1998 1999 @param power_method Specifies which method of power control to 2000 use. By default "RPM" will be used. Valid values 2001 are the strings "RPM", "manual", "servoj10". 2002 2003 """ 2004 if power_method in (self.POWER_CONTROL_SERVO, 2005 self.POWER_CONTROL_MANUAL): 2006 self.power_off(power_method=power_method) 2007 time.sleep(self._POWER_CYCLE_TIMEOUT) 2008 self.power_on(power_method=power_method) 2009 else: 2010 rpm_client.set_power(self.hostname, 'CYCLE') 2011 2012 2013 def get_platform(self): 2014 """Determine the correct platform label for this host. 2015 2016 @returns a string representing this host's platform. 2017 """ 2018 cmd = 'mosys platform model' 2019 result = self.run(command=cmd, ignore_status=True) 2020 if result.exit_status == 0: 2021 return result.stdout.strip() 2022 else: 2023 # $(mosys platform model) should support all platforms, but it 2024 # currently doesn't, so this reverts to parsing the fw 2025 # for any unsupported mosys platforms. 2026 crossystem = utils.Crossystem(self) 2027 crossystem.init() 2028 # Extract fwid value and use the leading part as the platform id. 2029 # fwid generally follow the format of {platform}.{firmware version} 2030 # Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z 2031 platform = crossystem.fwid().split('.')[0].lower() 2032 # Newer platforms start with 'Google_' while the older ones do not. 2033 return platform.replace('google_', '') 2034 2035 2036 def get_architecture(self): 2037 """Determine the correct architecture label for this host. 2038 2039 @returns a string representing this host's architecture. 2040 """ 2041 crossystem = utils.Crossystem(self) 2042 crossystem.init() 2043 return crossystem.arch() 2044 2045 2046 def get_chrome_version(self): 2047 """Gets the Chrome version number and milestone as strings. 2048 2049 Invokes "chrome --version" to get the version number and milestone. 2050 2051 @return A tuple (chrome_ver, milestone) where "chrome_ver" is the 2052 current Chrome version number as a string (in the form "W.X.Y.Z") 2053 and "milestone" is the first component of the version number 2054 (the "W" from "W.X.Y.Z"). If the version number cannot be parsed 2055 in the "W.X.Y.Z" format, the "chrome_ver" will be the full output 2056 of "chrome --version" and the milestone will be the empty string. 2057 2058 """ 2059 version_string = self.run(client_constants.CHROME_VERSION_COMMAND).stdout 2060 return utils.parse_chrome_version(version_string) 2061 2062 2063 def get_ec_version(self): 2064 """Get the ec version as strings. 2065 2066 @returns a string representing this host's ec version. 2067 """ 2068 command = 'mosys ec info -s fw_version' 2069 result = self.run(command, ignore_status=True) 2070 if result.exit_status != 0: 2071 return '' 2072 return result.stdout.strip() 2073 2074 2075 def get_firmware_version(self): 2076 """Get the firmware version as strings. 2077 2078 @returns a string representing this host's firmware version. 2079 """ 2080 crossystem = utils.Crossystem(self) 2081 crossystem.init() 2082 return crossystem.fwid() 2083 2084 2085 def get_hardware_revision(self): 2086 """Get the hardware revision as strings. 2087 2088 @returns a string representing this host's hardware revision. 2089 """ 2090 command = 'mosys platform version' 2091 result = self.run(command, ignore_status=True) 2092 if result.exit_status != 0: 2093 return '' 2094 return result.stdout.strip() 2095 2096 2097 def get_kernel_version(self): 2098 """Get the kernel version as strings. 2099 2100 @returns a string representing this host's kernel version. 2101 """ 2102 return self.run('uname -r').stdout.strip() 2103 2104 2105 def is_chrome_switch_present(self, switch): 2106 """Returns True if the specified switch was provided to Chrome. 2107 2108 @param switch The chrome switch to search for. 2109 """ 2110 2111 command = 'pgrep -x -f -c "/opt/google/chrome/chrome.*%s.*"' % switch 2112 return self.run(command, ignore_status=True).exit_status == 0 2113 2114 2115 def oobe_triggers_update(self): 2116 """Returns True if this host has an OOBE flow during which 2117 it will perform an update check and perhaps an update. 2118 One example of such a flow is Hands-Off Zero-Touch Enrollment. 2119 As more such flows are developed, code handling them needs 2120 to be added here. 2121 2122 @return Boolean indicating whether this host's OOBE triggers an update. 2123 """ 2124 return self.is_chrome_switch_present( 2125 '--enterprise-enable-zero-touch-enrollment=hands-off') 2126 2127 2128 # TODO(kevcheng): change this to just return the board without the 2129 # 'board:' prefix and fix up all the callers. Also look into removing the 2130 # need for this method. 2131 def get_board(self): 2132 """Determine the correct board label for this host. 2133 2134 @returns a string representing this host's board. 2135 """ 2136 release_info = utils.parse_cmd_output('cat /etc/lsb-release', 2137 run_method=self.run) 2138 return (ds_constants.BOARD_PREFIX + 2139 release_info['CHROMEOS_RELEASE_BOARD']) 2140 2141 def get_channel(self): 2142 """Determine the correct channel label for this host. 2143 2144 @returns: a string represeting this host's build channel. 2145 (stable, dev, beta). None on fail. 2146 """ 2147 return lsbrelease_utils.get_chromeos_channel( 2148 lsb_release_content=self._get_lsb_release_content()) 2149 2150 def has_lightsensor(self): 2151 """Determine the correct board label for this host. 2152 2153 @returns the string 'lightsensor' if this host has a lightsensor or 2154 None if it does not. 2155 """ 2156 search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % ( 2157 self._LIGHTSENSOR_SEARCH_DIR, '|'.join(self._LIGHTSENSOR_FILES)) 2158 try: 2159 # Run the search cmd following the symlinks. Stderr_tee is set to 2160 # None as there can be a symlink loop, but the command will still 2161 # execute correctly with a few messages printed to stderr. 2162 self.run(search_cmd, stdout_tee=None, stderr_tee=None) 2163 return 'lightsensor' 2164 except error.AutoservRunError: 2165 # egrep exited with a return code of 1 meaning none of the possible 2166 # lightsensor files existed. 2167 return None 2168 2169 2170 def has_bluetooth(self): 2171 """Determine the correct board label for this host. 2172 2173 @returns the string 'bluetooth' if this host has bluetooth or 2174 None if it does not. 2175 """ 2176 try: 2177 self.run('test -d /sys/class/bluetooth/hci0') 2178 # test exited with a return code of 0. 2179 return 'bluetooth' 2180 except error.AutoservRunError: 2181 # test exited with a return code 1 meaning the directory did not 2182 # exist. 2183 return None 2184 2185 2186 def get_accels(self): 2187 """ 2188 Determine the type of accelerometers on this host. 2189 2190 @returns a string representing this host's accelerometer type. 2191 At present, it only returns "accel:cros-ec", for accelerometers 2192 attached to a Chrome OS EC, or none, if no accelerometers. 2193 """ 2194 # Check to make sure we have ectool 2195 rv = self.run('which ectool', ignore_status=True) 2196 if rv.exit_status: 2197 logging.info("No ectool cmd found, assuming no EC accelerometers") 2198 return None 2199 2200 # Check that the EC supports the motionsense command 2201 rv = self.run('ectool motionsense', ignore_status=True) 2202 if rv.exit_status: 2203 logging.info("EC does not support motionsense command " 2204 "assuming no EC accelerometers") 2205 return None 2206 2207 # Check that EC motion sensors are active 2208 active = self.run('ectool motionsense active').stdout.split('\n') 2209 if active[0] == "0": 2210 logging.info("Motion sense inactive, assuming no EC accelerometers") 2211 return None 2212 2213 logging.info("EC accelerometers found") 2214 return 'accel:cros-ec' 2215 2216 2217 def has_chameleon(self): 2218 """Determine if a Chameleon connected to this host. 2219 2220 @returns a list containing two strings ('chameleon' and 2221 'chameleon:' + label, e.g. 'chameleon:hdmi') if this host 2222 has a Chameleon or None if it has not. 2223 """ 2224 if self._chameleon_host: 2225 return ['chameleon', 'chameleon:' + self.chameleon.get_label()] 2226 else: 2227 return None 2228 2229 2230 def has_loopback_dongle(self): 2231 """Determine if an audio loopback dongle is plugged to this host. 2232 2233 @returns 'audio_loopback_dongle' when there is an audio loopback dongle 2234 plugged to this host. 2235 None when there is no audio loopback dongle 2236 plugged to this host. 2237 """ 2238 nodes_info = self.run(command=cras_utils.get_cras_nodes_cmd(), 2239 ignore_status=True).stdout 2240 if (cras_utils.node_type_is_plugged('HEADPHONE', nodes_info) and 2241 cras_utils.node_type_is_plugged('MIC', nodes_info)): 2242 return 'audio_loopback_dongle' 2243 else: 2244 return None 2245 2246 2247 def get_power_supply(self): 2248 """ 2249 Determine what type of power supply the host has 2250 2251 @returns a string representing this host's power supply. 2252 'power:battery' when the device has a battery intended for 2253 extended use 2254 'power:AC_primary' when the device has a battery not intended 2255 for extended use (for moving the machine, etc) 2256 'power:AC_only' when the device has no battery at all. 2257 """ 2258 psu = self.run(command='mosys psu type', ignore_status=True) 2259 if psu.exit_status: 2260 # The psu command for mosys is not included for all platforms. The 2261 # assumption is that the device will have a battery if the command 2262 # is not found. 2263 return 'power:battery' 2264 2265 psu_str = psu.stdout.strip() 2266 if psu_str == 'unknown': 2267 return None 2268 2269 return 'power:%s' % psu_str 2270 2271 2272 def get_storage(self): 2273 """ 2274 Determine the type of boot device for this host. 2275 2276 Determine if the internal device is SCSI or dw_mmc device. 2277 Then check that it is SSD or HDD or eMMC or something else. 2278 2279 @returns a string representing this host's internal device type. 2280 'storage:ssd' when internal device is solid state drive 2281 'storage:hdd' when internal device is hard disk drive 2282 'storage:mmc' when internal device is mmc drive 2283 None When internal device is something else or 2284 when we are unable to determine the type 2285 """ 2286 # The output should be /dev/mmcblk* for SD/eMMC or /dev/sd* for scsi 2287 rootdev_cmd = ' '.join(['. /usr/sbin/write_gpt.sh;', 2288 '. /usr/share/misc/chromeos-common.sh;', 2289 'load_base_vars;', 2290 'get_fixed_dst_drive']) 2291 rootdev = self.run(command=rootdev_cmd, ignore_status=True) 2292 if rootdev.exit_status: 2293 logging.info("Fail to run %s", rootdev_cmd) 2294 return None 2295 rootdev_str = rootdev.stdout.strip() 2296 2297 if not rootdev_str: 2298 return None 2299 2300 rootdev_base = os.path.basename(rootdev_str) 2301 2302 mmc_pattern = '/dev/mmcblk[0-9]' 2303 if re.match(mmc_pattern, rootdev_str): 2304 # Use type to determine if the internal device is eMMC or somthing 2305 # else. We can assume that MMC is always an internal device. 2306 type_cmd = 'cat /sys/block/%s/device/type' % rootdev_base 2307 type = self.run(command=type_cmd, ignore_status=True) 2308 if type.exit_status: 2309 logging.info("Fail to run %s", type_cmd) 2310 return None 2311 type_str = type.stdout.strip() 2312 2313 if type_str == 'MMC': 2314 return 'storage:mmc' 2315 2316 scsi_pattern = '/dev/sd[a-z]+' 2317 if re.match(scsi_pattern, rootdev.stdout): 2318 # Read symlink for /sys/block/sd* to determine if the internal 2319 # device is connected via ata or usb. 2320 link_cmd = 'readlink /sys/block/%s' % rootdev_base 2321 link = self.run(command=link_cmd, ignore_status=True) 2322 if link.exit_status: 2323 logging.info("Fail to run %s", link_cmd) 2324 return None 2325 link_str = link.stdout.strip() 2326 if 'usb' in link_str: 2327 return None 2328 2329 # Read rotation to determine if the internal device is ssd or hdd. 2330 rotate_cmd = str('cat /sys/block/%s/queue/rotational' 2331 % rootdev_base) 2332 rotate = self.run(command=rotate_cmd, ignore_status=True) 2333 if rotate.exit_status: 2334 logging.info("Fail to run %s", rotate_cmd) 2335 return None 2336 rotate_str = rotate.stdout.strip() 2337 2338 rotate_dict = {'0':'storage:ssd', '1':'storage:hdd'} 2339 return rotate_dict.get(rotate_str) 2340 2341 # All other internal device / error case will always fall here 2342 return None 2343 2344 2345 def get_servo(self): 2346 """Determine if the host has a servo attached. 2347 2348 If the host has a working servo attached, it should have a servo label. 2349 2350 @return: string 'servo' if the host has servo attached. Otherwise, 2351 returns None. 2352 """ 2353 return 'servo' if self._servo_host else None 2354 2355 2356 def get_video_labels(self): 2357 """Run /usr/local/bin/avtest_label_detect to get a list of video labels. 2358 2359 Sample output of avtest_label_detect: 2360 Detected label: hw_video_acc_vp8 2361 Detected label: webcam 2362 2363 @return: A list of labels detected by tool avtest_label_detect. 2364 """ 2365 try: 2366 result = self.run('/usr/local/bin/avtest_label_detect').stdout 2367 return re.findall('^Detected label: (\w+)$', result, re.M) 2368 except error.AutoservRunError: 2369 # The tool is not installed. 2370 return [] 2371 2372 2373 def is_video_glitch_detection_supported(self): 2374 """ Determine if a board under test is supported for video glitch 2375 detection tests. 2376 2377 @return: 'video_glitch_detection' if board is supported, None otherwise. 2378 """ 2379 board = self.get_board().replace(ds_constants.BOARD_PREFIX, '') 2380 2381 if board in video_test_constants.SUPPORTED_BOARDS: 2382 return 'video_glitch_detection' 2383 2384 return None 2385 2386 2387 def get_touch(self): 2388 """ 2389 Determine whether board under test has a touchpad or touchscreen. 2390 2391 @return: A list of some combination of 'touchscreen' and 'touchpad', 2392 depending on what is present on the device. 2393 2394 """ 2395 labels = [] 2396 looking_for = ['touchpad', 'touchscreen'] 2397 player = input_playback.InputPlayback() 2398 input_events = self.run('ls /dev/input/event*').stdout.strip().split() 2399 filename = '/tmp/touch_labels' 2400 for event in input_events: 2401 self.run('evtest %s > %s' % (event, filename), timeout=1, 2402 ignore_timeout=True) 2403 properties = self.run('cat %s' % filename).stdout 2404 input_type = player._determine_input_type(properties) 2405 if input_type in looking_for: 2406 labels.append(input_type) 2407 looking_for.remove(input_type) 2408 if len(looking_for) == 0: 2409 break 2410 self.run('rm %s' % filename) 2411 2412 return labels 2413 2414 2415 def has_internal_display(self): 2416 """Determine if the device under test is equipped with an internal 2417 display. 2418 2419 @return: 'internal_display' if one is present; None otherwise. 2420 """ 2421 from autotest_lib.client.cros.graphics import graphics_utils 2422 from autotest_lib.client.common_lib import utils as common_utils 2423 2424 def __system_output(cmd): 2425 return self.run(cmd).stdout 2426 2427 def __read_file(remote_path): 2428 return self.run('cat %s' % remote_path).stdout 2429 2430 # Hijack the necessary client functions so that we can take advantage 2431 # of the client lib here. 2432 # FIXME: find a less hacky way than this 2433 original_system_output = utils.system_output 2434 original_read_file = common_utils.read_file 2435 utils.system_output = __system_output 2436 common_utils.read_file = __read_file 2437 try: 2438 return ('internal_display' if graphics_utils.has_internal_display() 2439 else None) 2440 finally: 2441 utils.system_output = original_system_output 2442 common_utils.read_file = original_read_file 2443 2444 2445 def is_boot_from_usb(self): 2446 """Check if DUT is boot from USB. 2447 2448 @return: True if DUT is boot from usb. 2449 """ 2450 device = self.run('rootdev -s -d').stdout.strip() 2451 removable = int(self.run('cat /sys/block/%s/removable' % 2452 os.path.basename(device)).stdout.strip()) 2453 return removable == 1 2454 2455 2456 def read_from_meminfo(self, key): 2457 """Return the memory info from /proc/meminfo 2458 2459 @param key: meminfo requested 2460 2461 @return the memory value as a string 2462 2463 """ 2464 meminfo = self.run('grep %s /proc/meminfo' % key).stdout.strip() 2465 logging.debug('%s', meminfo) 2466 return int(re.search(r'\d+', meminfo).group(0)) 2467 2468 2469 def get_cpu_arch(self): 2470 """Returns CPU arch of the device. 2471 2472 @return CPU architecture of the DUT. 2473 """ 2474 # Add CPUs by following logic in client/bin/utils.py. 2475 if self.run("grep '^flags.*:.* lm .*' /proc/cpuinfo", 2476 ignore_status=True).stdout: 2477 return 'x86_64' 2478 if self.run("grep -Ei 'ARM|CPU implementer' /proc/cpuinfo", 2479 ignore_status=True).stdout: 2480 return 'arm' 2481 return 'i386' 2482 2483 2484 def get_board_type(self): 2485 """ 2486 Get the DUT's device type from /etc/lsb-release. 2487 DEVICETYPE can be one of CHROMEBOX, CHROMEBASE, CHROMEBOOK or more. 2488 2489 @return value of DEVICETYPE param from lsb-release. 2490 """ 2491 device_type = self.run('grep DEVICETYPE /etc/lsb-release', 2492 ignore_status=True).stdout 2493 if device_type: 2494 return device_type.split('=')[-1].strip() 2495 return '' 2496 2497 2498 def get_arc_version(self): 2499 """Return ARC version installed on the DUT. 2500 2501 @returns ARC version as string if the CrOS build has ARC, else None. 2502 """ 2503 arc_version = self.run('grep CHROMEOS_ARC_VERSION /etc/lsb-release', 2504 ignore_status=True).stdout 2505 if arc_version: 2506 return arc_version.split('=')[-1].strip() 2507 return None 2508 2509 2510 def get_os_type(self): 2511 return 'cros' 2512 2513 2514 def enable_adb_testing(self): 2515 """Mark this host as an adb tester.""" 2516 self.run('touch %s' % constants.ANDROID_TESTER_FILEFLAG) 2517 2518 2519 def get_labels(self): 2520 """Return the detected labels on the host.""" 2521 return self.labels.get_labels(self) 2522 2523 2524 def update_labels(self): 2525 """Update the labels on the host.""" 2526 self.labels.update_labels(self) 2527