• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import re
8import sys
9import time
10
11import common
12from autotest_lib.client.bin import utils
13from autotest_lib.client.common_lib import autotemp
14from autotest_lib.client.common_lib import error
15from autotest_lib.client.common_lib import global_config
16from autotest_lib.client.common_lib import hosts
17from autotest_lib.client.common_lib import lsbrelease_utils
18from autotest_lib.client.common_lib.cros import autoupdater
19from autotest_lib.client.common_lib.cros import dev_server
20from autotest_lib.client.common_lib.cros import retry
21from autotest_lib.client.cros import constants as client_constants
22from autotest_lib.client.cros import cros_ui
23from autotest_lib.client.cros.audio import cras_utils
24from autotest_lib.client.cros.input_playback import input_playback
25from autotest_lib.client.cros.video import constants as video_test_constants
26from autotest_lib.server import afe_utils
27from autotest_lib.server import autoserv_parser
28from autotest_lib.server import autotest
29from autotest_lib.server import constants
30from autotest_lib.server import utils as server_utils
31from autotest_lib.server.cros import provision
32from autotest_lib.server.cros.dynamic_suite import constants as ds_constants
33from autotest_lib.server.cros.dynamic_suite import tools, frontend_wrappers
34from autotest_lib.server.cros.faft.config.config import Config as FAFTConfig
35from autotest_lib.server.cros.servo import firmware_programmer
36from autotest_lib.server.cros.servo import plankton
37from autotest_lib.server.hosts import abstract_ssh
38from autotest_lib.server.hosts import base_label
39from autotest_lib.server.hosts import cros_label
40from autotest_lib.server.hosts import chameleon_host
41from autotest_lib.server.hosts import cros_repair
42from autotest_lib.server.hosts import plankton_host
43from autotest_lib.server.hosts import servo_host
44from autotest_lib.site_utils.rpm_control_system import rpm_client
45
46# In case cros_host is being ran via SSP on an older Moblab version with an
47# older chromite version.
48try:
49    from chromite.lib import metrics
50except ImportError:
51    metrics =  utils.metrics_mock
52
53
54CONFIG = global_config.global_config
55ENABLE_DEVSERVER_TRIGGER_AUTO_UPDATE = CONFIG.get_config_value(
56        'CROS', 'enable_devserver_trigger_auto_update', type=bool,
57        default=False)
58
59
60class FactoryImageCheckerException(error.AutoservError):
61    """Exception raised when an image is a factory image."""
62    pass
63
64
65class CrosHost(abstract_ssh.AbstractSSHHost):
66    """Chromium OS specific subclass of Host."""
67
68    VERSION_PREFIX = provision.CROS_VERSION_PREFIX
69
70    _parser = autoserv_parser.autoserv_parser
71    _AFE = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
72    support_devserver_provision = ENABLE_DEVSERVER_TRIGGER_AUTO_UPDATE
73
74    # Timeout values (in seconds) associated with various Chrome OS
75    # state changes.
76    #
77    # In general, a good rule of thumb is that the timeout can be up
78    # to twice the typical measured value on the slowest platform.
79    # The times here have not necessarily been empirically tested to
80    # meet this criterion.
81    #
82    # SLEEP_TIMEOUT:  Time to allow for suspend to memory.
83    # RESUME_TIMEOUT: Time to allow for resume after suspend, plus
84    #   time to restart the netwowrk.
85    # SHUTDOWN_TIMEOUT: Time to allow for shut down.
86    # BOOT_TIMEOUT: Time to allow for boot from power off.  Among
87    #   other things, this must account for the 30 second dev-mode
88    #   screen delay, time to start the network on the DUT, and the
89    #   ssh timeout of 120 seconds.
90    # USB_BOOT_TIMEOUT: Time to allow for boot from a USB device,
91    #   including the 30 second dev-mode delay and time to start the
92    #   network.
93    # INSTALL_TIMEOUT: Time to allow for chromeos-install.
94    # POWERWASH_BOOT_TIMEOUT: Time to allow for a reboot that
95    #   includes powerwash.
96
97    SLEEP_TIMEOUT = 2
98    RESUME_TIMEOUT = 10
99    SHUTDOWN_TIMEOUT = 10
100    BOOT_TIMEOUT = 150
101    USB_BOOT_TIMEOUT = 300
102    INSTALL_TIMEOUT = 480
103    POWERWASH_BOOT_TIMEOUT = 60
104
105    # Minimum OS version that supports server side packaging. Older builds may
106    # not have server side package built or with Autotest code change to support
107    # server-side packaging.
108    MIN_VERSION_SUPPORT_SSP = CONFIG.get_config_value(
109            'AUTOSERV', 'min_version_support_ssp', type=int)
110
111    # REBOOT_TIMEOUT: How long to wait for a reboot.
112    #
113    # We have a long timeout to ensure we don't flakily fail due to other
114    # issues. Shorter timeouts are vetted in platform_RebootAfterUpdate.
115    # TODO(sbasi - crbug.com/276094) Restore to 5 mins once the 'host did not
116    # return from reboot' bug is solved.
117    REBOOT_TIMEOUT = 480
118
119    # _USB_POWER_TIMEOUT: Time to allow for USB to power toggle ON and OFF.
120    # _POWER_CYCLE_TIMEOUT: Time to allow for manual power cycle.
121    _USB_POWER_TIMEOUT = 5
122    _POWER_CYCLE_TIMEOUT = 10
123
124    _LAB_MACHINE_FILE = '/mnt/stateful_partition/.labmachine'
125    _RPM_HOSTNAME_REGEX = ('chromeos(\d+)(-row(\d+))?-rack(\d+[a-z]*)'
126                           '-host(\d+)')
127    _LIGHTSENSOR_FILES = [ "in_illuminance0_input",
128                           "in_illuminance_input",
129                           "in_illuminance0_raw",
130                           "in_illuminance_raw",
131                           "illuminance0_input"]
132    _LIGHTSENSOR_SEARCH_DIR = '/sys/bus/iio/devices'
133
134    # Constants used in ping_wait_up() and ping_wait_down().
135    #
136    # _PING_WAIT_COUNT is the approximate number of polling
137    # cycles to use when waiting for a host state change.
138    #
139    # _PING_STATUS_DOWN and _PING_STATUS_UP are names used
140    # for arguments to the internal _ping_wait_for_status()
141    # method.
142    _PING_WAIT_COUNT = 40
143    _PING_STATUS_DOWN = False
144    _PING_STATUS_UP = True
145
146    # Allowed values for the power_method argument.
147
148    # POWER_CONTROL_RPM: Passed as default arg for power_off/on/cycle() methods.
149    # POWER_CONTROL_SERVO: Used in set_power() and power_cycle() methods.
150    # POWER_CONTROL_MANUAL: Used in set_power() and power_cycle() methods.
151    POWER_CONTROL_RPM = 'RPM'
152    POWER_CONTROL_SERVO = 'servoj10'
153    POWER_CONTROL_MANUAL = 'manual'
154
155    POWER_CONTROL_VALID_ARGS = (POWER_CONTROL_RPM,
156                                POWER_CONTROL_SERVO,
157                                POWER_CONTROL_MANUAL)
158
159    _RPM_OUTLET_CHANGED = 'outlet_changed'
160
161    # URL pattern to download firmware image.
162    _FW_IMAGE_URL_PATTERN = CONFIG.get_config_value(
163            'CROS', 'firmware_url_pattern', type=str)
164
165
166    # A flag file to indicate provision failures.  The file is created
167    # at the start of any AU procedure (see `machine_install()`).  The
168    # file's location in stateful means that on successul update it will
169    # be removed.  Thus, if this file exists, it indicates that we've
170    # tried and failed in a previous attempt to update.
171    PROVISION_FAILED = '/var/tmp/provision_failed'
172
173
174    @staticmethod
175    def check_host(host, timeout=10):
176        """
177        Check if the given host is a chrome-os host.
178
179        @param host: An ssh host representing a device.
180        @param timeout: The timeout for the run command.
181
182        @return: True if the host device is chromeos.
183
184        """
185        try:
186            result = host.run(
187                    'grep -q CHROMEOS /etc/lsb-release && '
188                    '! test -f /mnt/stateful_partition/.android_tester && '
189                    '! grep -q moblab /etc/lsb-release',
190                    ignore_status=True, timeout=timeout)
191            if result.exit_status == 0:
192                lsb_release_content = host.run(
193                    'grep CHROMEOS_RELEASE_BOARD /etc/lsb-release',
194                    timeout=timeout).stdout
195                return not (
196                    lsbrelease_utils.is_jetstream(
197                        lsb_release_content=lsb_release_content) or
198                    lsbrelease_utils.is_gce_board(
199                        lsb_release_content=lsb_release_content))
200
201        except (error.AutoservRunError, error.AutoservSSHTimeout):
202            return False
203
204        return False
205
206
207    @staticmethod
208    def get_chameleon_arguments(args_dict):
209        """Extract chameleon options from `args_dict` and return the result.
210
211        Recommended usage:
212        ~~~~~~~~
213            args_dict = utils.args_to_dict(args)
214            chameleon_args = hosts.CrosHost.get_chameleon_arguments(args_dict)
215            host = hosts.create_host(machine, chameleon_args=chameleon_args)
216        ~~~~~~~~
217
218        @param args_dict Dictionary from which to extract the chameleon
219          arguments.
220        """
221        return {key: args_dict[key]
222                for key in ('chameleon_host', 'chameleon_port')
223                if key in args_dict}
224
225
226    @staticmethod
227    def get_plankton_arguments(args_dict):
228        """Extract chameleon options from `args_dict` and return the result.
229
230        Recommended usage:
231        ~~~~~~~~
232            args_dict = utils.args_to_dict(args)
233            plankton_args = hosts.CrosHost.get_plankton_arguments(args_dict)
234            host = hosts.create_host(machine, plankton_args=plankton_args)
235        ~~~~~~~~
236
237        @param args_dict Dictionary from which to extract the plankton
238          arguments.
239        """
240        return {key: args_dict[key]
241                for key in ('plankton_host', 'plankton_port')
242                if key in args_dict}
243
244
245    @staticmethod
246    def get_servo_arguments(args_dict):
247        """Extract servo options from `args_dict` and return the result.
248
249        Recommended usage:
250        ~~~~~~~~
251            args_dict = utils.args_to_dict(args)
252            servo_args = hosts.CrosHost.get_servo_arguments(args_dict)
253            host = hosts.create_host(machine, servo_args=servo_args)
254        ~~~~~~~~
255
256        @param args_dict Dictionary from which to extract the servo
257          arguments.
258        """
259        servo_attrs = (servo_host.SERVO_HOST_ATTR,
260                       servo_host.SERVO_PORT_ATTR,
261                       servo_host.SERVO_BOARD_ATTR)
262        servo_args = {key: args_dict[key]
263                      for key in servo_attrs
264                      if key in args_dict}
265        return (
266            None
267            if servo_host.SERVO_HOST_ATTR in servo_args
268                and not servo_args[servo_host.SERVO_HOST_ATTR]
269            else servo_args)
270
271
272    def _initialize(self, hostname, chameleon_args=None, servo_args=None,
273                    plankton_args=None, try_lab_servo=False,
274                    try_servo_repair=False,
275                    ssh_verbosity_flag='', ssh_options='',
276                    *args, **dargs):
277        """Initialize superclasses, |self.chameleon|, and |self.servo|.
278
279        This method will attempt to create the test-assistant object
280        (chameleon/servo) when it is needed by the test. Check
281        the docstring of chameleon_host.create_chameleon_host and
282        servo_host.create_servo_host for how this is determined.
283
284        @param hostname: Hostname of the dut.
285        @param chameleon_args: A dictionary that contains args for creating
286                               a ChameleonHost. See chameleon_host for details.
287        @param servo_args: A dictionary that contains args for creating
288                           a ServoHost object. See servo_host for details.
289        @param try_lab_servo: When true, indicates that an attempt should
290                              be made to create a ServoHost for a DUT in
291                              the test lab, even if not required by
292                              `servo_args`. See servo_host for details.
293        @param try_servo_repair: If a servo host is created, check it
294                              with `repair()` rather than `verify()`.
295                              See servo_host for details.
296        @param ssh_verbosity_flag: String, to pass to the ssh command to control
297                                   verbosity.
298        @param ssh_options: String, other ssh options to pass to the ssh
299                            command.
300        """
301        super(CrosHost, self)._initialize(hostname=hostname,
302                                          *args, **dargs)
303        self._repair_strategy = cros_repair.create_cros_repair_strategy()
304        self.labels = base_label.LabelRetriever(cros_label.CROS_LABELS)
305        # self.env is a dictionary of environment variable settings
306        # to be exported for commands run on the host.
307        # LIBC_FATAL_STDERR_ can be useful for diagnosing certain
308        # errors that might happen.
309        self.env['LIBC_FATAL_STDERR_'] = '1'
310        self._ssh_verbosity_flag = ssh_verbosity_flag
311        self._ssh_options = ssh_options
312        # TODO(fdeng): We need to simplify the
313        # process of servo and servo_host initialization.
314        # crbug.com/298432
315        self._servo_host = servo_host.create_servo_host(
316                dut=self, servo_args=servo_args,
317                try_lab_servo=try_lab_servo,
318                try_servo_repair=try_servo_repair)
319        if self._servo_host is not None:
320            self.servo = self._servo_host.get_servo()
321        else:
322            self.servo = None
323
324        # TODO(waihong): Do the simplication on Chameleon too.
325        self._chameleon_host = chameleon_host.create_chameleon_host(
326                dut=self.hostname, chameleon_args=chameleon_args)
327        # Add plankton host if plankton args were added on command line
328        self._plankton_host = plankton_host.create_plankton_host(plankton_args)
329
330        if self._chameleon_host:
331            self.chameleon = self._chameleon_host.create_chameleon_board()
332        else:
333            self.chameleon = None
334
335        if self._plankton_host:
336            self.plankton_servo = self._plankton_host.get_servo()
337            logging.info('plankton_servo: %r', self.plankton_servo)
338            # Create the plankton object used to access the ec uart
339            self.plankton = plankton.Plankton(self.plankton_servo,
340                    self._plankton_host.get_servod_server_proxy())
341        else:
342            self.plankton = None
343
344
345    def _get_cros_repair_image_name(self):
346        info = self.host_info_store.get()
347        if info.board is None:
348            raise error.AutoservError('Cannot obtain repair image name. '
349                                      'No board label value found')
350
351        return afe_utils.get_stable_cros_image_name(info.board)
352
353
354    def host_version_prefix(self, image):
355        """Return version label prefix.
356
357        In case the CrOS provisioning version is something other than the
358        standard CrOS version e.g. CrOS TH version, this function will
359        find the prefix from provision.py.
360
361        @param image: The image name to find its version prefix.
362        @returns: A prefix string for the image type.
363        """
364        return provision.get_version_label_prefix(image)
365
366
367    def verify_job_repo_url(self, tag=''):
368        """
369        Make sure job_repo_url of this host is valid.
370
371        Eg: The job_repo_url "http://lmn.cd.ab.xyx:8080/static/\
372        lumpy-release/R29-4279.0.0/autotest/packages" claims to have the
373        autotest package for lumpy-release/R29-4279.0.0. If this isn't the case,
374        download and extract it. If the devserver embedded in the url is
375        unresponsive, update the job_repo_url of the host after staging it on
376        another devserver.
377
378        @param job_repo_url: A url pointing to the devserver where the autotest
379            package for this build should be staged.
380        @param tag: The tag from the server job, in the format
381                    <job_id>-<user>/<hostname>, or <hostless> for a server job.
382
383        @raises DevServerException: If we could not resolve a devserver.
384        @raises AutoservError: If we're unable to save the new job_repo_url as
385            a result of choosing a new devserver because the old one failed to
386            respond to a health check.
387        @raises urllib2.URLError: If the devserver embedded in job_repo_url
388                                  doesn't respond within the timeout.
389        """
390        info = self.host_info_store.get()
391        job_repo_url = info.attributes.get(ds_constants.JOB_REPO_URL, '')
392        if not job_repo_url:
393            logging.warning('No job repo url set on host %s', self.hostname)
394            return
395
396        logging.info('Verifying job repo url %s', job_repo_url)
397        devserver_url, image_name = tools.get_devserver_build_from_package_url(
398            job_repo_url)
399
400        ds = dev_server.ImageServer(devserver_url)
401
402        logging.info('Staging autotest artifacts for %s on devserver %s',
403            image_name, ds.url())
404
405        start_time = time.time()
406        ds.stage_artifacts(image_name, ['autotest_packages'])
407        stage_time = time.time() - start_time
408
409        # Record how much of the verification time comes from a devserver
410        # restage. If we're doing things right we should not see multiple
411        # devservers for a given board/build/branch path.
412        try:
413            board, build_type, branch = server_utils.ParseBuildName(
414                                                image_name)[:3]
415        except server_utils.ParseBuildNameException:
416            pass
417        else:
418            devserver = devserver_url[
419                devserver_url.find('/') + 2:devserver_url.rfind(':')]
420            stats_key = {
421                'board': board,
422                'build_type': build_type,
423                'branch': branch,
424                'devserver': devserver.replace('.', '_'),
425            }
426
427            monarch_fields = {
428                'board': board,
429                'build_type': build_type,
430                # TODO(akeshet): To be consistent with most other metrics,
431                # consider changing the following field to be named
432                # 'milestone'.
433                'branch': branch,
434                'dev_server': devserver,
435            }
436            metrics.Counter(
437                    'chromeos/autotest/provision/verify_url'
438                    ).increment(fields=monarch_fields)
439            metrics.SecondsDistribution(
440                    'chromeos/autotest/provision/verify_url_duration'
441                    ).add(stage_time, fields=monarch_fields)
442
443
444    def stage_server_side_package(self, image=None):
445        """Stage autotest server-side package on devserver.
446
447        @param image: Full path of an OS image to install or a build name.
448
449        @return: A url to the autotest server-side package.
450
451        @raise: error.AutoservError if fail to locate the build to test with, or
452                fail to stage server-side package.
453        """
454        # If enable_drone_in_restricted_subnet is False, do not set hostname
455        # in devserver.resolve call, so a devserver in non-restricted subnet
456        # is picked to stage autotest server package for drone to download.
457        hostname = self.hostname
458        if not server_utils.ENABLE_DRONE_IN_RESTRICTED_SUBNET:
459            hostname = None
460        if image:
461            image_name = tools.get_build_from_image(image)
462            if not image_name:
463                raise error.AutoservError(
464                        'Failed to parse build name from %s' % image)
465            ds = dev_server.ImageServer.resolve(image_name, hostname)
466        else:
467            info = self.host_info_store.get()
468            job_repo_url = info.attributes.get(ds_constants.JOB_REPO_URL, '')
469            if job_repo_url:
470                devserver_url, image_name = (
471                    tools.get_devserver_build_from_package_url(job_repo_url))
472                # If enable_drone_in_restricted_subnet is True, use the
473                # existing devserver. Otherwise, resolve a new one in
474                # non-restricted subnet.
475                if server_utils.ENABLE_DRONE_IN_RESTRICTED_SUBNET:
476                    ds = dev_server.ImageServer(devserver_url)
477                else:
478                    ds = dev_server.ImageServer.resolve(image_name)
479            elif info.build is not None:
480                ds = dev_server.ImageServer.resolve(info.build, hostname)
481                image_name = info.build
482            else:
483                raise error.AutoservError(
484                        'Failed to stage server-side package. The host has '
485                        'no job_report_url attribute or version label.')
486
487        # Get the OS version of the build, for any build older than
488        # MIN_VERSION_SUPPORT_SSP, server side packaging is not supported.
489        match = re.match('.*/R\d+-(\d+)\.', image_name)
490        if match and int(match.group(1)) < self.MIN_VERSION_SUPPORT_SSP:
491            raise error.AutoservError(
492                    'Build %s is older than %s. Server side packaging is '
493                    'disabled.' % (image_name, self.MIN_VERSION_SUPPORT_SSP))
494
495        ds.stage_artifacts(image_name, ['autotest_server_package'])
496        return '%s/static/%s/%s' % (ds.url(), image_name,
497                                    'autotest_server_package.tar.bz2')
498
499
500    def _try_stateful_update(self, update_url, force_update, updater):
501        """Try to use stateful update to initialize DUT.
502
503        When DUT is already running the same version that machine_install
504        tries to install, stateful update is a much faster way to clean up
505        the DUT for testing, compared to a full reimage. It is implemeted
506        by calling autoupdater.run_update, but skipping updating root, as
507        updating the kernel is time consuming and not necessary.
508
509        @param update_url: url of the image.
510        @param force_update: Set to True to update the image even if the DUT
511            is running the same version.
512        @param updater: ChromiumOSUpdater instance used to update the DUT.
513        @returns: True if the DUT was updated with stateful update.
514
515        """
516        self.prepare_for_update()
517
518        # TODO(jrbarnette):  Yes, I hate this re.match() test case.
519        # It's better than the alternative:  see crbug.com/360944.
520        image_name = autoupdater.url_to_image_name(update_url)
521        release_pattern = r'^.*-release/R[0-9]+-[0-9]+\.[0-9]+\.0$'
522        if not re.match(release_pattern, image_name):
523            return False
524        if not updater.check_version():
525            return False
526        if not force_update:
527            logging.info('Canceling stateful update because the new and '
528                         'old versions are the same.')
529            return False
530        # Following folders should be rebuilt after stateful update.
531        # A test file is used to confirm each folder gets rebuilt after
532        # the stateful update.
533        folders_to_check = ['/var', '/home', '/mnt/stateful_partition']
534        test_file = '.test_file_to_be_deleted'
535        paths = [os.path.join(folder, test_file) for folder in folders_to_check]
536        self.run('touch %s' % ' '.join(paths))
537
538        updater.run_update(update_root=False)
539
540        # Reboot to complete stateful update.
541        self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
542
543        # After stateful update and a reboot, all of the test_files shouldn't
544        # exist any more. Otherwise the stateful update is failed.
545        return not any(
546            self.path_exists(os.path.join(folder, test_file))
547            for folder in folders_to_check)
548
549
550    def _post_update_processing(self, updater, expected_kernel=None):
551        """After the DUT is updated, confirm machine_install succeeded.
552
553        @param updater: ChromiumOSUpdater instance used to update the DUT.
554        @param expected_kernel: kernel expected to be active after reboot,
555            or `None` to skip rollback checking.
556
557        """
558        # Touch the lab machine file to leave a marker that
559        # distinguishes this image from other test images.
560        # Afterwards, we must re-run the autoreboot script because
561        # it depends on the _LAB_MACHINE_FILE.
562        autoreboot_cmd = ('FILE="%s" ; [ -f "$FILE" ] || '
563                          '( touch "$FILE" ; start autoreboot )')
564        self.run(autoreboot_cmd % self._LAB_MACHINE_FILE)
565        updater.verify_boot_expectations(
566                expected_kernel, rollback_message=
567                'Build %s failed to boot on %s; system rolled back to previous '
568                'build' % (updater.update_version, self.hostname))
569        # Check that we've got the build we meant to install.
570        if not updater.check_version_to_confirm_install():
571            raise autoupdater.ChromiumOSError(
572                'Failed to update %s to build %s; found build '
573                '%s instead' % (self.hostname,
574                                updater.update_version,
575                                self.get_release_version()))
576
577        logging.debug('Cleaning up old autotest directories.')
578        try:
579            installed_autodir = autotest.Autotest.get_installed_autodir(self)
580            self.run('rm -rf ' + installed_autodir)
581        except autotest.AutodirNotFoundError:
582            logging.debug('No autotest installed directory found.')
583
584
585    def _stage_image_for_update(self, image_name=None):
586        """Stage a build on a devserver and return the update_url and devserver.
587
588        @param image_name: a name like lumpy-release/R27-3837.0.0
589        @returns a tuple with an update URL like:
590            http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
591            and the devserver instance.
592        """
593        if not image_name:
594            image_name = self._get_cros_repair_image_name()
595        logging.info('Staging build for AU: %s', image_name)
596        devserver = dev_server.ImageServer.resolve(image_name, self.hostname)
597        devserver.trigger_download(image_name, synchronous=False)
598        return (tools.image_url_pattern() % (devserver.url(), image_name),
599                devserver)
600
601
602    def stage_image_for_servo(self, image_name=None):
603        """Stage a build on a devserver and return the update_url.
604
605        @param image_name: a name like lumpy-release/R27-3837.0.0
606        @returns an update URL like:
607            http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0
608        """
609        if not image_name:
610            image_name = self._get_cros_repair_image_name()
611        logging.info('Staging build for servo install: %s', image_name)
612        devserver = dev_server.ImageServer.resolve(image_name, self.hostname)
613        devserver.stage_artifacts(image_name, ['test_image'])
614        return devserver.get_test_image_url(image_name)
615
616
617    def stage_factory_image_for_servo(self, image_name):
618        """Stage a build on a devserver and return the update_url.
619
620        @param image_name: a name like <baord>/4262.204.0
621
622        @return: An update URL, eg:
623            http://<devserver>/static/canary-channel/\
624            <board>/4262.204.0/factory_test/chromiumos_factory_image.bin
625
626        @raises: ValueError if the factory artifact name is missing from
627                 the config.
628
629        """
630        if not image_name:
631            logging.error('Need an image_name to stage a factory image.')
632            return
633
634        factory_artifact = CONFIG.get_config_value(
635                'CROS', 'factory_artifact', type=str, default='')
636        if not factory_artifact:
637            raise ValueError('Cannot retrieve the factory artifact name from '
638                             'autotest config, and hence cannot stage factory '
639                             'artifacts.')
640
641        logging.info('Staging build for servo install: %s', image_name)
642        devserver = dev_server.ImageServer.resolve(image_name, self.hostname)
643        devserver.stage_artifacts(
644                image_name,
645                [factory_artifact],
646                archive_url=None)
647
648        return tools.factory_image_url_pattern() % (devserver.url(), image_name)
649
650
651    def _get_au_monarch_fields(self, devserver, build):
652        """Form monarch fields by given devserve & build for auto-update.
653
654        @param devserver: the devserver (ImageServer instance) for auto-update.
655        @param build: the build to be updated.
656
657        @return A dictionary of monach fields.
658        """
659        try:
660            board, build_type, milestone, _ = server_utils.ParseBuildName(build)
661        except server_utils.ParseBuildNameException:
662            logging.warning('Unable to parse build name %s. Something is '
663                            'likely broken, but will continue anyway.',
664                            build)
665            board, build_type, milestone = ('', '', '')
666
667        monarch_fields = {
668                'dev_server': devserver.hostname,
669                'board': board,
670                'build_type': build_type,
671                'milestone': milestone
672        }
673        return monarch_fields
674
675
676    def _retry_auto_update_with_new_devserver(self, build, last_devserver,
677                                              force_update, force_full_update,
678                                              force_original, quick_provision):
679        """Kick off auto-update by devserver and send metrics.
680
681        @param build: the build to update.
682        @param last_devserver: the last devserver that failed to provision.
683        @param force_update: see |machine_install_by_devserver|'s force_udpate
684                             for details.
685        @param force_full_update: see |machine_install_by_devserver|'s
686                                  force_full_update for details.
687        @param force_original: Whether to force stateful update with the
688                               original payload.
689        @param quick_provision: Attempt to use quick provision path first.
690
691        @return the result of |auto_update| in dev_server.
692        """
693        devserver = dev_server.resolve(
694                build, self.hostname, ban_list=[last_devserver.url()])
695        devserver.trigger_download(build, synchronous=False)
696        monarch_fields = self._get_au_monarch_fields(devserver, build)
697        logging.debug('Retry auto_update: resolved devserver for '
698                      'auto-update: %s', devserver.url())
699
700        # Add metrics
701        install_with_dev_counter = metrics.Counter(
702                'chromeos/autotest/provision/install_with_devserver')
703        install_with_dev_counter.increment(fields=monarch_fields)
704        c = metrics.Counter(
705                'chromeos/autotest/provision/retry_by_devserver')
706        monarch_fields['last_devserver'] = last_devserver.hostname
707        monarch_fields['host'] = self.hostname
708        c.increment(fields=monarch_fields)
709
710        # Won't retry auto_update in a retry of auto-update.
711        # In other words, we only retry auto-update once with a different
712        # devservers.
713        devserver.auto_update(
714                self.hostname, build,
715                original_board=self.get_board().replace(
716                        ds_constants.BOARD_PREFIX, ''),
717                original_release_version=self.get_release_version(),
718                log_dir=self.job.resultdir,
719                force_update=force_update,
720                full_update=force_full_update,
721                force_original=force_original,
722                quick_provision=quick_provision)
723
724
725    def machine_install_by_devserver(self, update_url=None, force_update=False,
726                    local_devserver=False, repair=False,
727                    force_full_update=False):
728        """Ultiize devserver to install the DUT.
729
730        (TODO) crbugs.com/627269: The logic in this function has some overlap
731        with those in function machine_install. The merge will be done later,
732        not in the same CL.
733
734        @param update_url: The update_url or build for the host to update.
735        @param force_update: Force an update even if the version installed
736                is the same. Default:False
737        @param local_devserver: Used by test_that to allow people to
738                use their local devserver. Default: False
739        @param repair: Forces update to repair image. Implies force_update.
740        @param force_full_update: If True, do not attempt to run stateful
741                update, force a full reimage. If False, try stateful update
742                first when the dut is already installed with the same version.
743        @raises autoupdater.ChromiumOSError
744
745        @returns A tuple of (image_name, host_attributes).
746                image_name is the name of image installed, e.g.,
747                veyron_jerry-release/R50-7871.0.0
748                host_attributes is a dictionary of (attribute, value), which
749                can be saved to afe_host_attributes table in database. This
750                method returns a dictionary with a single entry of
751                `job_repo_url`: repo_url, where repo_url is a devserver url to
752                autotest packages.
753        """
754        if repair:
755            update_url = self._get_cros_repair_image_name()
756            force_update = True
757
758        if not update_url and not self._parser.options.image:
759            raise error.AutoservError(
760                    'There is no update URL, nor a method to get one.')
761
762        if not update_url and self._parser.options.image:
763            update_url = self._parser.options.image
764
765
766        # Get build from parameter or AFE.
767        # If the build is not a URL, let devserver to stage it first.
768        # Otherwise, choose a devserver to trigger auto-update.
769        build = None
770        devserver = None
771        logging.debug('Resolving a devserver for auto-update')
772        previously_resolved = False
773        if update_url.startswith('http://'):
774            build = autoupdater.url_to_image_name(update_url)
775            previously_resolved = True
776        else:
777            build = update_url
778        devserver = dev_server.resolve(build, self.hostname)
779        server_name = devserver.hostname
780
781        monarch_fields = self._get_au_monarch_fields(devserver, build)
782
783        if previously_resolved:
784            # Make sure devserver for Auto-Update has staged the build.
785            if server_name not in update_url:
786              logging.debug('Resolved to devserver that does not match '
787                            'update_url. The previously resolved devserver '
788                            'must be unhealthy. Switching to use devserver %s,'
789                            ' and re-staging.',
790                            server_name)
791              logging.info('Staging build for AU: %s', update_url)
792              devserver.trigger_download(build, synchronous=False)
793              c = metrics.Counter(
794                  'chromeos/autotest/provision/failover_download')
795              c.increment(fields=monarch_fields)
796        else:
797            logging.info('Staging build for AU: %s', update_url)
798            devserver.trigger_download(build, synchronous=False)
799            c = metrics.Counter('chromeos/autotest/provision/trigger_download')
800            c.increment(fields=monarch_fields)
801
802        # Report provision stats.
803        install_with_dev_counter = metrics.Counter(
804                'chromeos/autotest/provision/install_with_devserver')
805        install_with_dev_counter.increment(fields=monarch_fields)
806        logging.debug('Resolved devserver for auto-update: %s', devserver.url())
807
808        # and other metrics from this function.
809        metrics.Counter('chromeos/autotest/provision/resolve'
810                        ).increment(fields=monarch_fields)
811
812        force_original = self.get_chromeos_release_milestone() is None
813
814        build_re = CONFIG.get_config_value(
815                'CROS', 'quick_provision_build_regex', type=str, default='')
816        quick_provision = (len(build_re) != 0 and
817                           re.match(build_re, build) is not None)
818
819        try:
820            devserver.auto_update(
821                    self.hostname, build,
822                    original_board=self.get_board().replace(
823                            ds_constants.BOARD_PREFIX, ''),
824                    original_release_version=self.get_release_version(),
825                    log_dir=self.job.resultdir,
826                    force_update=force_update,
827                    full_update=force_full_update,
828                    force_original=force_original,
829                    quick_provision=quick_provision)
830        except dev_server.RetryableProvisionException:
831            # It indicates that last provision failed due to devserver load
832            # issue, so another devserver is resolved to kick off provision
833            # job once again and only once.
834            logging.debug('Provision failed due to devserver issue,'
835                          'retry it with another devserver.')
836
837            # Check first whether this DUT is completely offline. If so, skip
838            # the following provision tries.
839            logging.debug('Checking whether host %s is online.', self.hostname)
840            if utils.ping(self.hostname, tries=1, deadline=1) == 0:
841                self._retry_auto_update_with_new_devserver(
842                        build, devserver, force_update, force_full_update,
843                        force_original, quick_provision)
844            else:
845                raise error.AutoservError(
846                        'No answer to ping from %s' % self.hostname)
847
848        # The reason to resolve a new devserver in function machine_install
849        # is mostly because that the update_url there may has a strange format,
850        # and it's hard to parse the devserver url from it.
851        # Since we already resolve a devserver to trigger auto-update, the same
852        # devserver is used to form JOB_REPO_URL here. Verified in local test.
853        repo_url = tools.get_package_url(devserver.url(), build)
854        return build, {ds_constants.JOB_REPO_URL: repo_url}
855
856
857    def prepare_for_update(self):
858        """Prepares the DUT for an update.
859
860        Subclasses may override this to perform any special actions
861        required before updating.
862        """
863        pass
864
865
866    def machine_install(self, update_url=None, force_update=False,
867                        local_devserver=False, repair=False,
868                        force_full_update=False):
869        """Install the DUT.
870
871        Use stateful update if the DUT is already running the same build.
872        Stateful update does not update kernel and tends to run much faster
873        than a full reimage. If the DUT is running a different build, or it
874        failed to do a stateful update, full update, including kernel update,
875        will be applied to the DUT.
876
877        Once a host enters machine_install its host attribute job_repo_url
878        (used for package install) will be removed and then updated.
879
880        @param update_url: The url to use for the update
881                pattern: http://$devserver:###/update/$build
882                If update_url is None and repair is True we will install the
883                stable image listed in afe_stable_versions table. If the table
884                is not setup, global_config value under CROS.stable_cros_version
885                will be used instead.
886        @param force_update: Force an update even if the version installed
887                is the same. Default:False
888        @param local_devserver: Used by test_that to allow people to
889                use their local devserver. Default: False
890        @param repair: Forces update to repair image. Implies force_update.
891        @param force_full_update: If True, do not attempt to run stateful
892                update, force a full reimage. If False, try stateful update
893                first when the dut is already installed with the same version.
894        @raises autoupdater.ChromiumOSError
895
896        @returns A tuple of (image_name, host_attributes).
897                image_name is the name of image installed, e.g.,
898                veyron_jerry-release/R50-7871.0.0
899                host_attributes is a dictionary of (attribute, value), which
900                can be saved to afe_host_attributes table in database. This
901                method returns a dictionary with a single entry of
902                `job_repo_url`: repo_url, where repo_url is a devserver url to
903                autotest packages.
904        """
905        devserver = None
906        if repair:
907            update_url, devserver = self._stage_image_for_update()
908            force_update = True
909
910        if not update_url and not self._parser.options.image:
911            raise error.AutoservError(
912                    'There is no update URL, nor a method to get one.')
913
914        if not update_url and self._parser.options.image:
915            # This is the base case where we have no given update URL i.e.
916            # dynamic suites logic etc. This is the most flexible case where we
917            # can serve an update from any of our fleet of devservers.
918            requested_build = self._parser.options.image
919            if not requested_build.startswith('http://'):
920                logging.debug('Update will be staged for this installation')
921                update_url, devserver = self._stage_image_for_update(
922                        requested_build)
923            else:
924                update_url = requested_build
925
926        logging.debug('Update URL is %s', update_url)
927
928        # Report provision stats.
929        server_name = dev_server.get_hostname(update_url)
930        (metrics.Counter('chromeos/autotest/provision/install')
931         .increment(fields={'devserver': server_name}))
932
933        # Create a file to indicate if provision fails. The file will be removed
934        # by stateful update or full install.
935        self.run('touch %s' % self.PROVISION_FAILED)
936
937        update_complete = False
938        updater = autoupdater.ChromiumOSUpdater(
939                 update_url, host=self, local_devserver=local_devserver)
940        if not force_full_update:
941            try:
942                # If the DUT is already running the same build, try stateful
943                # update first as it's much quicker than a full re-image.
944                update_complete = self._try_stateful_update(
945                        update_url, force_update, updater)
946            except Exception as e:
947                logging.exception(e)
948
949        inactive_kernel = None
950        if update_complete or (not force_update and updater.check_version()):
951            logging.info('Install complete without full update')
952        else:
953            logging.info('DUT requires full update.')
954            self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
955            self.prepare_for_update()
956
957            num_of_attempts = provision.FLAKY_DEVSERVER_ATTEMPTS
958
959            while num_of_attempts > 0:
960                num_of_attempts -= 1
961                try:
962                    updater.run_update()
963                except Exception:
964                    logging.warn('Autoupdate did not complete.')
965                    # Do additional check for the devserver health. Ideally,
966                    # the autoupdater.py could raise an exception when it
967                    # detected network flake but that would require
968                    # instrumenting the update engine and parsing it log.
969                    if (num_of_attempts <= 0 or
970                            devserver is None or
971                            dev_server.ImageServer.devserver_healthy(
972                                    devserver.url())):
973                        raise
974
975                    logging.warn('Devserver looks unhealthy. Trying another')
976                    update_url, devserver = self._stage_image_for_update(
977                            requested_build)
978                    logging.debug('New Update URL is %s', update_url)
979                    updater = autoupdater.ChromiumOSUpdater(
980                            update_url, host=self,
981                            local_devserver=local_devserver)
982                else:
983                    break
984
985            # Give it some time in case of IO issues.
986            time.sleep(10)
987
988            # Figure out active and inactive kernel.
989            active_kernel, inactive_kernel = updater.get_kernel_state()
990
991            # Ensure inactive kernel has higher priority than active.
992            if (updater.get_kernel_priority(inactive_kernel)
993                    < updater.get_kernel_priority(active_kernel)):
994                raise autoupdater.ChromiumOSError(
995                    'Update failed. The priority of the inactive kernel'
996                    ' partition is less than that of the active kernel'
997                    ' partition.')
998
999            # Updater has returned successfully; reboot the host.
1000            #
1001            # Regarding the 'crossystem' command: In some cases, the
1002            # TPM gets into a state such that it fails verification.
1003            # We don't know why.  However, this call papers over the
1004            # problem by clearing the TPM during the reboot.
1005            #
1006            # We ignore failures from 'crossystem'.  Although failure
1007            # here is unexpected, and could signal a bug, the point
1008            # of the exercise is to paper over problems; allowing
1009            # this to fail would defeat the purpose.
1010            self.run('crossystem clear_tpm_owner_request=1',
1011                     ignore_status=True)
1012            self.reboot(timeout=self.REBOOT_TIMEOUT, wait=True)
1013
1014        self._post_update_processing(updater, inactive_kernel)
1015        image_name = autoupdater.url_to_image_name(update_url)
1016        # update_url is different from devserver url needed to stage autotest
1017        # packages, therefore, resolve a new devserver url here.
1018        devserver_url = dev_server.ImageServer.resolve(image_name,
1019                                                       self.hostname).url()
1020        repo_url = tools.get_package_url(devserver_url, image_name)
1021        return image_name, {ds_constants.JOB_REPO_URL: repo_url}
1022
1023
1024    def _clear_fw_version_labels(self, rw_only):
1025        """Clear firmware version labels from the machine.
1026
1027        @param rw_only: True to only clear fwrw_version; otherewise, clear
1028                        both fwro_version and fwrw_version.
1029        """
1030        labels = self._AFE.get_labels(
1031                name__startswith=provision.FW_RW_VERSION_PREFIX,
1032                host__hostname=self.hostname)
1033        if not rw_only:
1034            labels = labels + self._AFE.get_labels(
1035                    name__startswith=provision.FW_RO_VERSION_PREFIX,
1036                    host__hostname=self.hostname)
1037        for label in labels:
1038            label.remove_hosts(hosts=[self.hostname])
1039
1040
1041    def _add_fw_version_label(self, build, rw_only):
1042        """Add firmware version label to the machine.
1043
1044        @param build: Build of firmware.
1045        @param rw_only: True to only add fwrw_version; otherwise, add both
1046                        fwro_version and fwrw_version.
1047
1048        """
1049        fw_label = provision.fwrw_version_to_label(build)
1050        self._AFE.run('label_add_hosts', id=fw_label, hosts=[self.hostname])
1051        if not rw_only:
1052            fw_label = provision.fwro_version_to_label(build)
1053            self._AFE.run('label_add_hosts', id=fw_label, hosts=[self.hostname])
1054
1055
1056    def firmware_install(self, build=None, rw_only=False):
1057        """Install firmware to the DUT.
1058
1059        Use stateful update if the DUT is already running the same build.
1060        Stateful update does not update kernel and tends to run much faster
1061        than a full reimage. If the DUT is running a different build, or it
1062        failed to do a stateful update, full update, including kernel update,
1063        will be applied to the DUT.
1064
1065        Once a host enters firmware_install its fw[ro|rw]_version label will
1066        be removed. After the firmware is updated successfully, a new
1067        fw[ro|rw]_version label will be added to the host.
1068
1069        @param build: The build version to which we want to provision the
1070                      firmware of the machine,
1071                      e.g. 'link-firmware/R22-2695.1.144'.
1072        @param rw_only: True to only install firmware to its RW portions. Keep
1073                        the RO portions unchanged.
1074
1075        TODO(dshi): After bug 381718 is fixed, update here with corresponding
1076                    exceptions that could be raised.
1077
1078        """
1079        if not self.servo:
1080            raise error.TestError('Host %s does not have servo.' %
1081                                  self.hostname)
1082
1083        # Get the DUT board name from servod.
1084        board = self.servo.get_board()
1085
1086        # If build is not set, try to install firmware from stable CrOS.
1087        if not build:
1088            build = afe_utils.get_stable_faft_version(board)
1089            if not build:
1090                raise error.TestError(
1091                        'Failed to find stable firmware build for %s.',
1092                        self.hostname)
1093            logging.info('Will install firmware from build %s.', build)
1094
1095        config = FAFTConfig(board)
1096        if config.use_u_boot:
1097            ap_image = 'image-%s.bin' % board
1098        else: # Depthcharge platform
1099            ap_image = 'image.bin'
1100        ec_image = 'ec.bin'
1101        ds = dev_server.ImageServer.resolve(build, self.hostname)
1102        ds.stage_artifacts(build, ['firmware'])
1103
1104        tmpd = autotemp.tempdir(unique_id='fwimage')
1105        try:
1106            fwurl = self._FW_IMAGE_URL_PATTERN % (ds.url(), build)
1107            local_tarball = os.path.join(tmpd.name, os.path.basename(fwurl))
1108            ds.download_file(fwurl, local_tarball)
1109
1110            self._clear_fw_version_labels(rw_only)
1111            if config.chrome_ec:
1112                logging.info('Will re-program EC %snow', 'RW ' if rw_only else '')
1113                server_utils.system('tar xf %s -C %s %s' %
1114                                    (local_tarball, tmpd.name, ec_image),
1115                                    timeout=60)
1116                self.servo.program_ec(os.path.join(tmpd.name, ec_image), rw_only)
1117            else:
1118                logging.info('Not a Chrome EC, ignore re-programing it')
1119            logging.info('Will re-program BIOS %snow', 'RW ' if rw_only else '')
1120            server_utils.system('tar xf %s -C %s %s' %
1121                                (local_tarball, tmpd.name, ap_image),
1122                                timeout=60)
1123            self.servo.program_bios(os.path.join(tmpd.name, ap_image), rw_only)
1124            self.servo.get_power_state_controller().reset()
1125            time.sleep(self.servo.BOOT_DELAY)
1126            if utils.host_is_in_lab_zone(self.hostname):
1127                self._add_fw_version_label(build, rw_only)
1128        finally:
1129            tmpd.clean()
1130
1131
1132    def program_base_ec(self, image_path):
1133        """Program Base EC on DUT with the given image.
1134
1135        @param image_path: a string, file name of the EC image to program
1136                           on the DUT.
1137
1138        """
1139        dest_path = os.path.join('/tmp', os.path.basename(image_path))
1140        self.send_file(image_path, dest_path)
1141        programmer = firmware_programmer.ProgrammerDfu(self.servo, self)
1142        programmer.program_ec(dest_path)
1143
1144
1145    def show_update_engine_log(self):
1146        """Output update engine log."""
1147        logging.debug('Dumping %s', client_constants.UPDATE_ENGINE_LOG)
1148        self.run('cat %s' % client_constants.UPDATE_ENGINE_LOG)
1149
1150
1151    def servo_install(self, image_url=None, usb_boot_timeout=USB_BOOT_TIMEOUT,
1152                      install_timeout=INSTALL_TIMEOUT):
1153        """
1154        Re-install the OS on the DUT by:
1155        1) installing a test image on a USB storage device attached to the Servo
1156                board,
1157        2) booting that image in recovery mode, and then
1158        3) installing the image with chromeos-install.
1159
1160        @param image_url: If specified use as the url to install on the DUT.
1161                otherwise boot the currently staged image on the USB stick.
1162        @param usb_boot_timeout: The usb_boot_timeout to use during reimage.
1163                Factory images need a longer usb_boot_timeout than regular
1164                cros images.
1165        @param install_timeout: The timeout to use when installing the chromeos
1166                image. Factory images need a longer install_timeout.
1167
1168        @raises AutoservError if the image fails to boot.
1169
1170        """
1171        logging.info('Downloading image to USB, then booting from it. Usb boot '
1172                     'timeout = %s', usb_boot_timeout)
1173        with metrics.SecondsTimer(
1174                'chromeos/autotest/provision/servo_install/boot_duration'):
1175            self.servo.install_recovery_image(image_url)
1176            if not self.wait_up(timeout=usb_boot_timeout):
1177                raise hosts.AutoservRepairError(
1178                        'DUT failed to boot from USB after %d seconds' %
1179                        usb_boot_timeout)
1180
1181        # The new chromeos-tpm-recovery has been merged since R44-7073.0.0.
1182        # In old CrOS images, this command fails. Skip the error.
1183        logging.info('Resetting the TPM status')
1184        try:
1185            self.run('chromeos-tpm-recovery')
1186        except error.AutoservRunError:
1187            logging.warn('chromeos-tpm-recovery is too old.')
1188
1189
1190        with metrics.SecondsTimer(
1191                'chromeos/autotest/provision/servo_install/install_duration'):
1192            logging.info('Installing image through chromeos-install.')
1193            self.run('chromeos-install --yes', timeout=install_timeout)
1194            self.halt()
1195
1196        logging.info('Power cycling DUT through servo.')
1197        self.servo.get_power_state_controller().power_off()
1198        self.servo.switch_usbkey('off')
1199        # N.B. The Servo API requires that we use power_on() here
1200        # for two reasons:
1201        #  1) After turning on a DUT in recovery mode, you must turn
1202        #     it off and then on with power_on() once more to
1203        #     disable recovery mode (this is a Parrot specific
1204        #     requirement).
1205        #  2) After power_off(), the only way to turn on is with
1206        #     power_on() (this is a Storm specific requirement).
1207        self.servo.get_power_state_controller().power_on()
1208
1209        logging.info('Waiting for DUT to come back up.')
1210        if not self.wait_up(timeout=self.BOOT_TIMEOUT):
1211            raise error.AutoservError('DUT failed to reboot installed '
1212                                      'test image after %d seconds' %
1213                                      self.BOOT_TIMEOUT)
1214
1215
1216    def repair_servo(self):
1217        """
1218        Confirm that servo is initialized and verified.
1219
1220        If the servo object is missing, attempt to repair the servo
1221        host.  Repair failures are passed back to the caller.
1222
1223        @raise AutoservError: If there is no servo host for this CrOS
1224                              host.
1225        """
1226        if self.servo:
1227            return
1228        if not self._servo_host:
1229            raise error.AutoservError('No servo host for %s.' %
1230                                      self.hostname)
1231        self._servo_host.repair()
1232        self.servo = self._servo_host.get_servo()
1233
1234
1235    def repair(self):
1236        """Attempt to get the DUT to pass `self.verify()`.
1237
1238        This overrides the base class function for repair; it does
1239        not call back to the parent class, but instead relies on
1240        `self._repair_strategy` to coordinate the verification and
1241        repair steps needed to get the DUT working.
1242        """
1243        self._repair_strategy.repair(self)
1244        # Sometimes, hosts with certain ethernet dongles get stuck in a
1245        # bad network state where they're reachable from this code, but
1246        # not from the devservers during provisioning.  Rebooting the
1247        # DUT fixes it.
1248        #
1249        # TODO(jrbarnette):  Ideally, we'd get rid of the problem
1250        # dongles, and drop this code.  Failing that, we could be smart
1251        # enough not to reboot if repair rebooted the DUT (e.g. by
1252        # looking at DUT uptime after repair completes).
1253
1254        self.reboot()
1255
1256
1257    def close(self):
1258        """Close connection."""
1259        super(CrosHost, self).close()
1260        if self._chameleon_host:
1261            self._chameleon_host.close()
1262
1263        if self._servo_host:
1264            self._servo_host.close()
1265
1266
1267    def get_power_supply_info(self):
1268        """Get the output of power_supply_info.
1269
1270        power_supply_info outputs the info of each power supply, e.g.,
1271        Device: Line Power
1272          online:                  no
1273          type:                    Mains
1274          voltage (V):             0
1275          current (A):             0
1276        Device: Battery
1277          state:                   Discharging
1278          percentage:              95.9276
1279          technology:              Li-ion
1280
1281        Above output shows two devices, Line Power and Battery, with details of
1282        each device listed. This function parses the output into a dictionary,
1283        with key being the device name, and value being a dictionary of details
1284        of the device info.
1285
1286        @return: The dictionary of power_supply_info, e.g.,
1287                 {'Line Power': {'online': 'yes', 'type': 'main'},
1288                  'Battery': {'vendor': 'xyz', 'percentage': '100'}}
1289        @raise error.AutoservRunError if power_supply_info tool is not found in
1290               the DUT. Caller should handle this error to avoid false failure
1291               on verification.
1292        """
1293        result = self.run('power_supply_info').stdout.strip()
1294        info = {}
1295        device_name = None
1296        device_info = {}
1297        for line in result.split('\n'):
1298            pair = [v.strip() for v in line.split(':')]
1299            if len(pair) != 2:
1300                continue
1301            if pair[0] == 'Device':
1302                if device_name:
1303                    info[device_name] = device_info
1304                device_name = pair[1]
1305                device_info = {}
1306            else:
1307                device_info[pair[0]] = pair[1]
1308        if device_name and not device_name in info:
1309            info[device_name] = device_info
1310        return info
1311
1312
1313    def get_battery_percentage(self):
1314        """Get the battery percentage.
1315
1316        @return: The percentage of battery level, value range from 0-100. Return
1317                 None if the battery info cannot be retrieved.
1318        """
1319        try:
1320            info = self.get_power_supply_info()
1321            logging.info(info)
1322            return float(info['Battery']['percentage'])
1323        except (KeyError, ValueError, error.AutoservRunError):
1324            return None
1325
1326
1327    def is_ac_connected(self):
1328        """Check if the dut has power adapter connected and charging.
1329
1330        @return: True if power adapter is connected and charging.
1331        """
1332        try:
1333            info = self.get_power_supply_info()
1334            return info['Line Power']['online'] == 'yes'
1335        except (KeyError, error.AutoservRunError):
1336            return None
1337
1338
1339    def _cleanup_poweron(self):
1340        """Special cleanup method to make sure hosts always get power back."""
1341        afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
1342        hosts = afe.get_hosts(hostname=self.hostname)
1343        if not hosts or not (self._RPM_OUTLET_CHANGED in
1344                             hosts[0].attributes):
1345            return
1346        logging.debug('This host has recently interacted with the RPM'
1347                      ' Infrastructure. Ensuring power is on.')
1348        try:
1349            self.power_on()
1350            afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None,
1351                                   hostname=self.hostname)
1352        except rpm_client.RemotePowerException:
1353            logging.error('Failed to turn Power On for this host after '
1354                          'cleanup through the RPM Infrastructure.')
1355
1356            battery_percentage = self.get_battery_percentage()
1357            if battery_percentage and battery_percentage < 50:
1358                raise
1359            elif self.is_ac_connected():
1360                logging.info('The device has power adapter connected and '
1361                             'charging. No need to try to turn RPM on '
1362                             'again.')
1363                afe.set_host_attribute(self._RPM_OUTLET_CHANGED, None,
1364                                       hostname=self.hostname)
1365            logging.info('Battery level is now at %s%%. The device may '
1366                         'still have enough power to run test, so no '
1367                         'exception will be raised.', battery_percentage)
1368
1369
1370    def _is_factory_image(self):
1371        """Checks if the image on the DUT is a factory image.
1372
1373        @return: True if the image on the DUT is a factory image.
1374                 False otherwise.
1375        """
1376        result = self.run('[ -f /root/.factory_test ]', ignore_status=True)
1377        return result.exit_status == 0
1378
1379
1380    def _restart_ui(self):
1381        """Restart the Chrome UI.
1382
1383        @raises: FactoryImageCheckerException for factory images, since
1384                 we cannot attempt to restart ui on them.
1385                 error.AutoservRunError for any other type of error that
1386                 occurs while restarting ui.
1387        """
1388        if self._is_factory_image():
1389            raise FactoryImageCheckerException('Cannot restart ui on factory '
1390                                               'images')
1391
1392        # TODO(jrbarnette):  The command to stop/start the ui job
1393        # should live inside cros_ui, too.  However that would seem
1394        # to imply interface changes to the existing start()/restart()
1395        # functions, which is a bridge too far (for now).
1396        prompt = cros_ui.get_chrome_session_ident(self)
1397        self.run('stop ui; start ui')
1398        cros_ui.wait_for_chrome_ready(prompt, self)
1399
1400
1401    def _get_lsb_release_content(self):
1402        """Return the content of lsb-release file of host."""
1403        return self.run(
1404                'cat "%s"' % client_constants.LSB_RELEASE).stdout.strip()
1405
1406
1407    def get_release_version(self):
1408        """Get the value of attribute CHROMEOS_RELEASE_VERSION from lsb-release.
1409
1410        @returns The version string in lsb-release, under attribute
1411                 CHROMEOS_RELEASE_VERSION.
1412        """
1413        return lsbrelease_utils.get_chromeos_release_version(
1414                lsb_release_content=self._get_lsb_release_content())
1415
1416
1417    def get_release_builder_path(self):
1418        """Get the value of CHROMEOS_RELEASE_BUILDER_PATH from lsb-release.
1419
1420        @returns The version string in lsb-release, under attribute
1421                 CHROMEOS_RELEASE_BUILDER_PATH.
1422        """
1423        return lsbrelease_utils.get_chromeos_release_builder_path(
1424                lsb_release_content=self._get_lsb_release_content())
1425
1426
1427    def get_chromeos_release_milestone(self):
1428        """Get the value of attribute CHROMEOS_RELEASE_BUILD_TYPE
1429        from lsb-release.
1430
1431        @returns The version string in lsb-release, under attribute
1432                 CHROMEOS_RELEASE_BUILD_TYPE.
1433        """
1434        return lsbrelease_utils.get_chromeos_release_milestone(
1435                lsb_release_content=self._get_lsb_release_content())
1436
1437
1438    def verify_cros_version_label(self):
1439        """ Make sure host's cros-version label match the actual image in dut.
1440
1441        Remove any cros-version: label that doesn't match that installed in
1442        the dut.
1443
1444        @param raise_error: Set to True to raise exception if any mismatch found
1445
1446        @raise error.AutoservError: If any mismatch between cros-version label
1447                                    and the build installed in dut is found.
1448        """
1449        labels = self._AFE.get_labels(
1450                name__startswith=ds_constants.VERSION_PREFIX,
1451                host__hostname=self.hostname)
1452        mismatch_found = False
1453        if labels:
1454            # Ask the DUT for its canonical image name.  This will be in
1455            # a form like this:  kevin-release/R66-10405.0.0
1456            release_builder_path = self.get_release_builder_path()
1457            host_list = [self.hostname]
1458            for label in labels:
1459                # Remove any cros-version label that does not match
1460                # the DUT's installed image.
1461                #
1462                # TODO(jrbarnette):  Tests sent to the `arc-presubmit`
1463                # pool install images matching the format above, but
1464                # then apply a label with `-cheetsth` appended.  Probably,
1465                # it's wrong for ARC presubmit testing to make that change,
1466                # but until it's fixed, this code specifically excuses that
1467                # behavior.
1468                build_version = label.name[len(ds_constants.VERSION_PREFIX):]
1469                if build_version.endswith('-cheetsth'):
1470                    build_version = build_version[:-len('-cheetsth')]
1471                if build_version != release_builder_path:
1472                    logging.warn(
1473                        'cros-version label "%s" does not match '
1474                        'release_builder_path %s. Removing the label.',
1475                        label.name, release_builder_path)
1476                    label.remove_hosts(hosts=host_list)
1477                    mismatch_found = True
1478        if mismatch_found:
1479            raise error.AutoservError('The host has wrong cros-version label.')
1480
1481
1482    def cleanup_services(self):
1483        """Reinitializes the device for cleanup.
1484
1485        Subclasses may override this to customize the cleanup method.
1486
1487        To indicate failure of the reset, the implementation may raise
1488        any of:
1489            error.AutoservRunError
1490            error.AutotestRunError
1491            FactoryImageCheckerException
1492
1493        @raises error.AutoservRunError
1494        @raises error.AutotestRunError
1495        @raises error.FactoryImageCheckerException
1496        """
1497        self._restart_ui()
1498
1499
1500    def cleanup(self):
1501        """Cleanup state on device."""
1502        self.run('rm -f %s' % client_constants.CLEANUP_LOGS_PAUSED_FILE)
1503        try:
1504            self.cleanup_services()
1505        except (error.AutotestRunError, error.AutoservRunError,
1506                FactoryImageCheckerException):
1507            logging.warning('Unable to restart ui, rebooting device.')
1508            # Since restarting the UI fails fall back to normal Autotest
1509            # cleanup routines, i.e. reboot the machine.
1510            super(CrosHost, self).cleanup()
1511        # Check if the rpm outlet was manipulated.
1512        if self.has_power():
1513            self._cleanup_poweron()
1514        self.verify_cros_version_label()
1515
1516
1517    def reboot(self, **dargs):
1518        """
1519        This function reboots the site host. The more generic
1520        RemoteHost.reboot() performs sync and sleeps for 5
1521        seconds. This is not necessary for Chrome OS devices as the
1522        sync should be finished in a short time during the reboot
1523        command.
1524        """
1525        if 'reboot_cmd' not in dargs:
1526            reboot_timeout = dargs.get('reboot_timeout', 10)
1527            dargs['reboot_cmd'] = ('sleep 1; '
1528                                   'reboot & sleep %d; '
1529                                   'reboot -f' % reboot_timeout)
1530        # Enable fastsync to avoid running extra sync commands before reboot.
1531        if 'fastsync' not in dargs:
1532            dargs['fastsync'] = True
1533
1534        # For purposes of logging reboot times:
1535        # Get the board name i.e. 'daisy_spring'
1536        board_fullname = self.get_board()
1537
1538        # Strip the prefix and add it to dargs.
1539        dargs['board'] = board_fullname[board_fullname.find(':')+1:]
1540        # Record who called us
1541        orig = sys._getframe(1).f_code
1542        metric_fields = {'board' : dargs['board'],
1543                         'dut_host_name' : self.hostname,
1544                         'success' : True}
1545        metric_debug_fields = {'board' : dargs['board'],
1546                               'caller' : "%s:%s" % (orig.co_filename, orig.co_name),
1547                               'success' : True,
1548                               'error' : ''}
1549
1550        t0 = time.time()
1551        try:
1552            super(CrosHost, self).reboot(**dargs)
1553        except Exception as e:
1554            metric_fields['success'] = False
1555            metric_debug_fields['success'] = False
1556            metric_debug_fields['error'] = type(e).__name__
1557            raise
1558        finally:
1559            duration = int(time.time() - t0)
1560            metrics.Counter(
1561                    'chromeos/autotest/autoserv/reboot_count').increment(
1562                    fields=metric_fields)
1563            metrics.Counter(
1564                    'chromeos/autotest/autoserv/reboot_debug').increment(
1565                    fields=metric_debug_fields)
1566            metrics.SecondsDistribution(
1567                    'chromeos/autotest/autoserv/reboot_duration').add(
1568                    duration, fields=metric_fields)
1569
1570
1571    def suspend(self, **dargs):
1572        """
1573        This function suspends the site host.
1574        """
1575        suspend_time = dargs.get('suspend_time', 60)
1576        dargs['timeout'] = suspend_time
1577        if 'suspend_cmd' not in dargs:
1578            dargs['suspend_cmd'] = ' && '.join([
1579                'echo 0 > /sys/class/rtc/rtc0/wakealarm',
1580                'echo +%d > /sys/class/rtc/rtc0/wakealarm' % suspend_time,
1581                'powerd_dbus_suspend --delay=0'])
1582        super(CrosHost, self).suspend(**dargs)
1583
1584
1585    def upstart_status(self, service_name):
1586        """Check the status of an upstart init script.
1587
1588        @param service_name: Service to look up.
1589
1590        @returns True if the service is running, False otherwise.
1591        """
1592        return 'start/running' in self.run('status %s' % service_name,
1593                                           ignore_status=True).stdout
1594
1595
1596    def verify_software(self):
1597        """Verify working software on a Chrome OS system.
1598
1599        Tests for the following conditions:
1600         1. All conditions tested by the parent version of this
1601            function.
1602         2. Sufficient space in /mnt/stateful_partition.
1603         3. Sufficient space in /mnt/stateful_partition/encrypted.
1604         4. update_engine answers a simple status request over DBus.
1605
1606        """
1607        super(CrosHost, self).verify_software()
1608        default_kilo_inodes_required = CONFIG.get_config_value(
1609                'SERVER', 'kilo_inodes_required', type=int, default=100)
1610        board = self.get_board().replace(ds_constants.BOARD_PREFIX, '')
1611        kilo_inodes_required = CONFIG.get_config_value(
1612                'SERVER', 'kilo_inodes_required_%s' % board,
1613                type=int, default=default_kilo_inodes_required)
1614        self.check_inodes('/mnt/stateful_partition', kilo_inodes_required)
1615        self.check_diskspace(
1616            '/mnt/stateful_partition',
1617            CONFIG.get_config_value(
1618                'SERVER', 'gb_diskspace_required', type=float,
1619                default=20.0))
1620        encrypted_stateful_path = '/mnt/stateful_partition/encrypted'
1621        # Not all targets build with encrypted stateful support.
1622        if self.path_exists(encrypted_stateful_path):
1623            self.check_diskspace(
1624                encrypted_stateful_path,
1625                CONFIG.get_config_value(
1626                    'SERVER', 'gb_encrypted_diskspace_required', type=float,
1627                    default=0.1))
1628
1629        self.wait_for_system_services()
1630
1631        # Factory images don't run update engine,
1632        # goofy controls dbus on these DUTs.
1633        if not self._is_factory_image():
1634            self.run('update_engine_client --status')
1635
1636        self.verify_cros_version_label()
1637
1638
1639    @retry.retry(error.AutoservError, timeout_min=5, delay_sec=10)
1640    def wait_for_system_services(self):
1641        """Waits for system-services to be running.
1642
1643        Sometimes, update_engine will take a while to update firmware, so we
1644        should give this some time to finish. See crbug.com/765686#c38 for
1645        details.
1646        """
1647        if not self.upstart_status('system-services'):
1648            raise error.AutoservError('Chrome failed to reach login. '
1649                                      'System services not running.')
1650
1651
1652    def verify(self):
1653        """Verify Chrome OS system is in good state."""
1654        self._repair_strategy.verify(self)
1655
1656
1657    def make_ssh_command(self, user='root', port=22, opts='', hosts_file=None,
1658                         connect_timeout=None, alive_interval=None,
1659                         alive_count_max=None, connection_attempts=None):
1660        """Override default make_ssh_command to use options tuned for Chrome OS.
1661
1662        Tuning changes:
1663          - ConnectTimeout=30; maximum of 30 seconds allowed for an SSH
1664          connection failure.  Consistency with remote_access.sh.
1665
1666          - ServerAliveInterval=900; which causes SSH to ping connection every
1667          900 seconds. In conjunction with ServerAliveCountMax ensures
1668          that if the connection dies, Autotest will bail out.
1669          Originally tried 60 secs, but saw frequent job ABORTS where
1670          the test completed successfully. Later increased from 180 seconds to
1671          900 seconds to account for tests where the DUT is suspended for
1672          longer periods of time.
1673
1674          - ServerAliveCountMax=3; consistency with remote_access.sh.
1675
1676          - ConnectAttempts=4; reduce flakiness in connection errors;
1677          consistency with remote_access.sh.
1678
1679          - UserKnownHostsFile=/dev/null; we don't care about the keys.
1680          Host keys change with every new installation, don't waste
1681          memory/space saving them.
1682
1683          - SSH protocol forced to 2; needed for ServerAliveInterval.
1684
1685        @param user User name to use for the ssh connection.
1686        @param port Port on the target host to use for ssh connection.
1687        @param opts Additional options to the ssh command.
1688        @param hosts_file Ignored.
1689        @param connect_timeout Ignored.
1690        @param alive_interval Ignored.
1691        @param alive_count_max Ignored.
1692        @param connection_attempts Ignored.
1693        """
1694        options = ' '.join([opts, '-o Protocol=2'])
1695        return super(CrosHost, self).make_ssh_command(
1696            user=user, port=port, opts=options, hosts_file='/dev/null',
1697            connect_timeout=30, alive_interval=900, alive_count_max=3,
1698            connection_attempts=4)
1699
1700
1701    def syslog(self, message, tag='autotest'):
1702        """Logs a message to syslog on host.
1703
1704        @param message String message to log into syslog
1705        @param tag String tag prefix for syslog
1706
1707        """
1708        self.run('logger -t "%s" "%s"' % (tag, message))
1709
1710
1711    def _ping_check_status(self, status):
1712        """Ping the host once, and return whether it has a given status.
1713
1714        @param status Check the ping status against this value.
1715        @return True iff `status` and the result of ping are the same
1716                (i.e. both True or both False).
1717
1718        """
1719        ping_val = utils.ping(self.hostname, tries=1, deadline=1)
1720        return not (status ^ (ping_val == 0))
1721
1722    def _ping_wait_for_status(self, status, timeout):
1723        """Wait for the host to have a given status (UP or DOWN).
1724
1725        Status is checked by polling.  Polling will not last longer
1726        than the number of seconds in `timeout`.  The polling
1727        interval will be long enough that only approximately
1728        _PING_WAIT_COUNT polling cycles will be executed, subject
1729        to a maximum interval of about one minute.
1730
1731        @param status Waiting will stop immediately if `ping` of the
1732                      host returns this status.
1733        @param timeout Poll for at most this many seconds.
1734        @return True iff the host status from `ping` matched the
1735                requested status at the time of return.
1736
1737        """
1738        # _ping_check_status() takes about 1 second, hence the
1739        # "- 1" in the formula below.
1740        # FIXME: if the ping command errors then _ping_check_status()
1741        # returns instantly. If timeout is also smaller than twice
1742        # _PING_WAIT_COUNT then the while loop below forks many
1743        # thousands of ping commands (see /tmp/test_that_results_XXXXX/
1744        # /results-1-logging_YYY.ZZZ/debug/autoserv.DEBUG) and hogs one
1745        # CPU core for 60 seconds.
1746        poll_interval = min(int(timeout / self._PING_WAIT_COUNT), 60) - 1
1747        end_time = time.time() + timeout
1748        while time.time() <= end_time:
1749            if self._ping_check_status(status):
1750                return True
1751            if poll_interval > 0:
1752                time.sleep(poll_interval)
1753
1754        # The last thing we did was sleep(poll_interval), so it may
1755        # have been too long since the last `ping`.  Check one more
1756        # time, just to be sure.
1757        return self._ping_check_status(status)
1758
1759    def ping_wait_up(self, timeout):
1760        """Wait for the host to respond to `ping`.
1761
1762        N.B.  This method is not a reliable substitute for
1763        `wait_up()`, because a host that responds to ping will not
1764        necessarily respond to ssh.  This method should only be used
1765        if the target DUT can be considered functional even if it
1766        can't be reached via ssh.
1767
1768        @param timeout Minimum time to allow before declaring the
1769                       host to be non-responsive.
1770        @return True iff the host answered to ping before the timeout.
1771
1772        """
1773        return self._ping_wait_for_status(self._PING_STATUS_UP, timeout)
1774
1775    def ping_wait_down(self, timeout):
1776        """Wait until the host no longer responds to `ping`.
1777
1778        This function can be used as a slightly faster version of
1779        `wait_down()`, by avoiding potentially long ssh timeouts.
1780
1781        @param timeout Minimum time to allow for the host to become
1782                       non-responsive.
1783        @return True iff the host quit answering ping before the
1784                timeout.
1785
1786        """
1787        return self._ping_wait_for_status(self._PING_STATUS_DOWN, timeout)
1788
1789    def test_wait_for_sleep(self, sleep_timeout=None):
1790        """Wait for the client to enter low-power sleep mode.
1791
1792        The test for "is asleep" can't distinguish a system that is
1793        powered off; to confirm that the unit was asleep, it is
1794        necessary to force resume, and then call
1795        `test_wait_for_resume()`.
1796
1797        This function is expected to be called from a test as part
1798        of a sequence like the following:
1799
1800        ~~~~~~~~
1801            boot_id = host.get_boot_id()
1802            # trigger sleep on the host
1803            host.test_wait_for_sleep()
1804            # trigger resume on the host
1805            host.test_wait_for_resume(boot_id)
1806        ~~~~~~~~
1807
1808        @param sleep_timeout time limit in seconds to allow the host sleep.
1809
1810        @exception TestFail The host did not go to sleep within
1811                            the allowed time.
1812        """
1813        if sleep_timeout is None:
1814            sleep_timeout = self.SLEEP_TIMEOUT
1815
1816        if not self.ping_wait_down(timeout=sleep_timeout):
1817            raise error.TestFail(
1818                'client failed to sleep after %d seconds' % sleep_timeout)
1819
1820
1821    def test_wait_for_resume(self, old_boot_id, resume_timeout=None):
1822        """Wait for the client to resume from low-power sleep mode.
1823
1824        The `old_boot_id` parameter should be the value from
1825        `get_boot_id()` obtained prior to entering sleep mode.  A
1826        `TestFail` exception is raised if the boot id changes.
1827
1828        See @ref test_wait_for_sleep for more on this function's
1829        usage.
1830
1831        @param old_boot_id A boot id value obtained before the
1832                               target host went to sleep.
1833        @param resume_timeout time limit in seconds to allow the host up.
1834
1835        @exception TestFail The host did not respond within the
1836                            allowed time.
1837        @exception TestFail The host responded, but the boot id test
1838                            indicated a reboot rather than a sleep
1839                            cycle.
1840        """
1841        if resume_timeout is None:
1842            resume_timeout = self.RESUME_TIMEOUT
1843
1844        if not self.wait_up(timeout=resume_timeout):
1845            raise error.TestFail(
1846                'client failed to resume from sleep after %d seconds' %
1847                    resume_timeout)
1848        else:
1849            new_boot_id = self.get_boot_id()
1850            if new_boot_id != old_boot_id:
1851                logging.error('client rebooted (old boot %s, new boot %s)',
1852                              old_boot_id, new_boot_id)
1853                raise error.TestFail(
1854                    'client rebooted, but sleep was expected')
1855
1856
1857    def test_wait_for_shutdown(self, shutdown_timeout=None):
1858        """Wait for the client to shut down.
1859
1860        The test for "has shut down" can't distinguish a system that
1861        is merely asleep; to confirm that the unit was down, it is
1862        necessary to force boot, and then call test_wait_for_boot().
1863
1864        This function is expected to be called from a test as part
1865        of a sequence like the following:
1866
1867        ~~~~~~~~
1868            boot_id = host.get_boot_id()
1869            # trigger shutdown on the host
1870            host.test_wait_for_shutdown()
1871            # trigger boot on the host
1872            host.test_wait_for_boot(boot_id)
1873        ~~~~~~~~
1874
1875        @param shutdown_timeout time limit in seconds to allow the host down.
1876        @exception TestFail The host did not shut down within the
1877                            allowed time.
1878        """
1879        if shutdown_timeout is None:
1880            shutdown_timeout = self.SHUTDOWN_TIMEOUT
1881
1882        if not self.ping_wait_down(timeout=shutdown_timeout):
1883            raise error.TestFail(
1884                'client failed to shut down after %d seconds' %
1885                    shutdown_timeout)
1886
1887
1888    def test_wait_for_boot(self, old_boot_id=None):
1889        """Wait for the client to boot from cold power.
1890
1891        The `old_boot_id` parameter should be the value from
1892        `get_boot_id()` obtained prior to shutting down.  A
1893        `TestFail` exception is raised if the boot id does not
1894        change.  The boot id test is omitted if `old_boot_id` is not
1895        specified.
1896
1897        See @ref test_wait_for_shutdown for more on this function's
1898        usage.
1899
1900        @param old_boot_id A boot id value obtained before the
1901                               shut down.
1902
1903        @exception TestFail The host did not respond within the
1904                            allowed time.
1905        @exception TestFail The host responded, but the boot id test
1906                            indicated that there was no reboot.
1907        """
1908        if not self.wait_up(timeout=self.REBOOT_TIMEOUT):
1909            raise error.TestFail(
1910                'client failed to reboot after %d seconds' %
1911                    self.REBOOT_TIMEOUT)
1912        elif old_boot_id:
1913            if self.get_boot_id() == old_boot_id:
1914                logging.error('client not rebooted (boot %s)',
1915                              old_boot_id)
1916                raise error.TestFail(
1917                    'client is back up, but did not reboot')
1918
1919
1920    @staticmethod
1921    def check_for_rpm_support(hostname):
1922        """For a given hostname, return whether or not it is powered by an RPM.
1923
1924        @param hostname: hostname to check for rpm support.
1925
1926        @return None if this host does not follows the defined naming format
1927                for RPM powered DUT's in the lab. If it does follow the format,
1928                it returns a regular expression MatchObject instead.
1929        """
1930        return re.match(CrosHost._RPM_HOSTNAME_REGEX, hostname)
1931
1932
1933    def has_power(self):
1934        """For this host, return whether or not it is powered by an RPM.
1935
1936        @return True if this host is in the CROS lab and follows the defined
1937                naming format.
1938        """
1939        return CrosHost.check_for_rpm_support(self.hostname)
1940
1941
1942    def _set_power(self, state, power_method):
1943        """Sets the power to the host via RPM, Servo or manual.
1944
1945        @param state Specifies which power state to set to DUT
1946        @param power_method Specifies which method of power control to
1947                            use. By default "RPM" will be used. Valid values
1948                            are the strings "RPM", "manual", "servoj10".
1949
1950        """
1951        ACCEPTABLE_STATES = ['ON', 'OFF']
1952
1953        if state.upper() not in ACCEPTABLE_STATES:
1954            raise error.TestError('State must be one of: %s.'
1955                                   % (ACCEPTABLE_STATES,))
1956
1957        if power_method == self.POWER_CONTROL_SERVO:
1958            logging.info('Setting servo port J10 to %s', state)
1959            self.servo.set('prtctl3_pwren', state.lower())
1960            time.sleep(self._USB_POWER_TIMEOUT)
1961        elif power_method == self.POWER_CONTROL_MANUAL:
1962            logging.info('You have %d seconds to set the AC power to %s.',
1963                         self._POWER_CYCLE_TIMEOUT, state)
1964            time.sleep(self._POWER_CYCLE_TIMEOUT)
1965        else:
1966            if not self.has_power():
1967                raise error.TestFail('DUT does not have RPM connected.')
1968            afe = frontend_wrappers.RetryingAFE(timeout_min=5, delay_sec=10)
1969            afe.set_host_attribute(self._RPM_OUTLET_CHANGED, True,
1970                                   hostname=self.hostname)
1971            rpm_client.set_power(self.hostname, state.upper(), timeout_mins=5)
1972
1973
1974    def power_off(self, power_method=POWER_CONTROL_RPM):
1975        """Turn off power to this host via RPM, Servo or manual.
1976
1977        @param power_method Specifies which method of power control to
1978                            use. By default "RPM" will be used. Valid values
1979                            are the strings "RPM", "manual", "servoj10".
1980
1981        """
1982        self._set_power('OFF', power_method)
1983
1984
1985    def power_on(self, power_method=POWER_CONTROL_RPM):
1986        """Turn on power to this host via RPM, Servo or manual.
1987
1988        @param power_method Specifies which method of power control to
1989                            use. By default "RPM" will be used. Valid values
1990                            are the strings "RPM", "manual", "servoj10".
1991
1992        """
1993        self._set_power('ON', power_method)
1994
1995
1996    def power_cycle(self, power_method=POWER_CONTROL_RPM):
1997        """Cycle power to this host by turning it OFF, then ON.
1998
1999        @param power_method Specifies which method of power control to
2000                            use. By default "RPM" will be used. Valid values
2001                            are the strings "RPM", "manual", "servoj10".
2002
2003        """
2004        if power_method in (self.POWER_CONTROL_SERVO,
2005                            self.POWER_CONTROL_MANUAL):
2006            self.power_off(power_method=power_method)
2007            time.sleep(self._POWER_CYCLE_TIMEOUT)
2008            self.power_on(power_method=power_method)
2009        else:
2010            rpm_client.set_power(self.hostname, 'CYCLE')
2011
2012
2013    def get_platform(self):
2014        """Determine the correct platform label for this host.
2015
2016        @returns a string representing this host's platform.
2017        """
2018        cmd = 'mosys platform model'
2019        result = self.run(command=cmd, ignore_status=True)
2020        if result.exit_status == 0:
2021            return result.stdout.strip()
2022        else:
2023            # $(mosys platform model) should support all platforms, but it
2024            # currently doesn't, so this reverts to parsing the fw
2025            # for any unsupported mosys platforms.
2026            crossystem = utils.Crossystem(self)
2027            crossystem.init()
2028            # Extract fwid value and use the leading part as the platform id.
2029            # fwid generally follow the format of {platform}.{firmware version}
2030            # Example: Alex.X.YYY.Z or Google_Alex.X.YYY.Z
2031            platform = crossystem.fwid().split('.')[0].lower()
2032            # Newer platforms start with 'Google_' while the older ones do not.
2033            return platform.replace('google_', '')
2034
2035
2036    def get_architecture(self):
2037        """Determine the correct architecture label for this host.
2038
2039        @returns a string representing this host's architecture.
2040        """
2041        crossystem = utils.Crossystem(self)
2042        crossystem.init()
2043        return crossystem.arch()
2044
2045
2046    def get_chrome_version(self):
2047        """Gets the Chrome version number and milestone as strings.
2048
2049        Invokes "chrome --version" to get the version number and milestone.
2050
2051        @return A tuple (chrome_ver, milestone) where "chrome_ver" is the
2052            current Chrome version number as a string (in the form "W.X.Y.Z")
2053            and "milestone" is the first component of the version number
2054            (the "W" from "W.X.Y.Z").  If the version number cannot be parsed
2055            in the "W.X.Y.Z" format, the "chrome_ver" will be the full output
2056            of "chrome --version" and the milestone will be the empty string.
2057
2058        """
2059        version_string = self.run(client_constants.CHROME_VERSION_COMMAND).stdout
2060        return utils.parse_chrome_version(version_string)
2061
2062
2063    def get_ec_version(self):
2064        """Get the ec version as strings.
2065
2066        @returns a string representing this host's ec version.
2067        """
2068        command = 'mosys ec info -s fw_version'
2069        result = self.run(command, ignore_status=True)
2070        if result.exit_status != 0:
2071            return ''
2072        return result.stdout.strip()
2073
2074
2075    def get_firmware_version(self):
2076        """Get the firmware version as strings.
2077
2078        @returns a string representing this host's firmware version.
2079        """
2080        crossystem = utils.Crossystem(self)
2081        crossystem.init()
2082        return crossystem.fwid()
2083
2084
2085    def get_hardware_revision(self):
2086        """Get the hardware revision as strings.
2087
2088        @returns a string representing this host's hardware revision.
2089        """
2090        command = 'mosys platform version'
2091        result = self.run(command, ignore_status=True)
2092        if result.exit_status != 0:
2093            return ''
2094        return result.stdout.strip()
2095
2096
2097    def get_kernel_version(self):
2098        """Get the kernel version as strings.
2099
2100        @returns a string representing this host's kernel version.
2101        """
2102        return self.run('uname -r').stdout.strip()
2103
2104
2105    def is_chrome_switch_present(self, switch):
2106        """Returns True if the specified switch was provided to Chrome.
2107
2108        @param switch The chrome switch to search for.
2109        """
2110
2111        command = 'pgrep -x -f -c "/opt/google/chrome/chrome.*%s.*"' % switch
2112        return self.run(command, ignore_status=True).exit_status == 0
2113
2114
2115    def oobe_triggers_update(self):
2116        """Returns True if this host has an OOBE flow during which
2117        it will perform an update check and perhaps an update.
2118        One example of such a flow is Hands-Off Zero-Touch Enrollment.
2119        As more such flows are developed, code handling them needs
2120        to be added here.
2121
2122        @return Boolean indicating whether this host's OOBE triggers an update.
2123        """
2124        return self.is_chrome_switch_present(
2125            '--enterprise-enable-zero-touch-enrollment=hands-off')
2126
2127
2128    # TODO(kevcheng): change this to just return the board without the
2129    # 'board:' prefix and fix up all the callers.  Also look into removing the
2130    # need for this method.
2131    def get_board(self):
2132        """Determine the correct board label for this host.
2133
2134        @returns a string representing this host's board.
2135        """
2136        release_info = utils.parse_cmd_output('cat /etc/lsb-release',
2137                                              run_method=self.run)
2138        return (ds_constants.BOARD_PREFIX +
2139                release_info['CHROMEOS_RELEASE_BOARD'])
2140
2141    def get_channel(self):
2142        """Determine the correct channel label for this host.
2143
2144        @returns: a string represeting this host's build channel.
2145                  (stable, dev, beta). None on fail.
2146        """
2147        return lsbrelease_utils.get_chromeos_channel(
2148                lsb_release_content=self._get_lsb_release_content())
2149
2150    def has_lightsensor(self):
2151        """Determine the correct board label for this host.
2152
2153        @returns the string 'lightsensor' if this host has a lightsensor or
2154                 None if it does not.
2155        """
2156        search_cmd = "find -L %s -maxdepth 4 | egrep '%s'" % (
2157            self._LIGHTSENSOR_SEARCH_DIR, '|'.join(self._LIGHTSENSOR_FILES))
2158        try:
2159            # Run the search cmd following the symlinks. Stderr_tee is set to
2160            # None as there can be a symlink loop, but the command will still
2161            # execute correctly with a few messages printed to stderr.
2162            self.run(search_cmd, stdout_tee=None, stderr_tee=None)
2163            return 'lightsensor'
2164        except error.AutoservRunError:
2165            # egrep exited with a return code of 1 meaning none of the possible
2166            # lightsensor files existed.
2167            return None
2168
2169
2170    def has_bluetooth(self):
2171        """Determine the correct board label for this host.
2172
2173        @returns the string 'bluetooth' if this host has bluetooth or
2174                 None if it does not.
2175        """
2176        try:
2177            self.run('test -d /sys/class/bluetooth/hci0')
2178            # test exited with a return code of 0.
2179            return 'bluetooth'
2180        except error.AutoservRunError:
2181            # test exited with a return code 1 meaning the directory did not
2182            # exist.
2183            return None
2184
2185
2186    def get_accels(self):
2187        """
2188        Determine the type of accelerometers on this host.
2189
2190        @returns a string representing this host's accelerometer type.
2191        At present, it only returns "accel:cros-ec", for accelerometers
2192        attached to a Chrome OS EC, or none, if no accelerometers.
2193        """
2194        # Check to make sure we have ectool
2195        rv = self.run('which ectool', ignore_status=True)
2196        if rv.exit_status:
2197            logging.info("No ectool cmd found, assuming no EC accelerometers")
2198            return None
2199
2200        # Check that the EC supports the motionsense command
2201        rv = self.run('ectool motionsense', ignore_status=True)
2202        if rv.exit_status:
2203            logging.info("EC does not support motionsense command "
2204                         "assuming no EC accelerometers")
2205            return None
2206
2207        # Check that EC motion sensors are active
2208        active = self.run('ectool motionsense active').stdout.split('\n')
2209        if active[0] == "0":
2210            logging.info("Motion sense inactive, assuming no EC accelerometers")
2211            return None
2212
2213        logging.info("EC accelerometers found")
2214        return 'accel:cros-ec'
2215
2216
2217    def has_chameleon(self):
2218        """Determine if a Chameleon connected to this host.
2219
2220        @returns a list containing two strings ('chameleon' and
2221                 'chameleon:' + label, e.g. 'chameleon:hdmi') if this host
2222                 has a Chameleon or None if it has not.
2223        """
2224        if self._chameleon_host:
2225            return ['chameleon', 'chameleon:' + self.chameleon.get_label()]
2226        else:
2227            return None
2228
2229
2230    def has_loopback_dongle(self):
2231        """Determine if an audio loopback dongle is plugged to this host.
2232
2233        @returns 'audio_loopback_dongle' when there is an audio loopback dongle
2234                                         plugged to this host.
2235                 None                    when there is no audio loopback dongle
2236                                         plugged to this host.
2237        """
2238        nodes_info = self.run(command=cras_utils.get_cras_nodes_cmd(),
2239                              ignore_status=True).stdout
2240        if (cras_utils.node_type_is_plugged('HEADPHONE', nodes_info) and
2241            cras_utils.node_type_is_plugged('MIC', nodes_info)):
2242                return 'audio_loopback_dongle'
2243        else:
2244                return None
2245
2246
2247    def get_power_supply(self):
2248        """
2249        Determine what type of power supply the host has
2250
2251        @returns a string representing this host's power supply.
2252                 'power:battery' when the device has a battery intended for
2253                        extended use
2254                 'power:AC_primary' when the device has a battery not intended
2255                        for extended use (for moving the machine, etc)
2256                 'power:AC_only' when the device has no battery at all.
2257        """
2258        psu = self.run(command='mosys psu type', ignore_status=True)
2259        if psu.exit_status:
2260            # The psu command for mosys is not included for all platforms. The
2261            # assumption is that the device will have a battery if the command
2262            # is not found.
2263            return 'power:battery'
2264
2265        psu_str = psu.stdout.strip()
2266        if psu_str == 'unknown':
2267            return None
2268
2269        return 'power:%s' % psu_str
2270
2271
2272    def get_storage(self):
2273        """
2274        Determine the type of boot device for this host.
2275
2276        Determine if the internal device is SCSI or dw_mmc device.
2277        Then check that it is SSD or HDD or eMMC or something else.
2278
2279        @returns a string representing this host's internal device type.
2280                 'storage:ssd' when internal device is solid state drive
2281                 'storage:hdd' when internal device is hard disk drive
2282                 'storage:mmc' when internal device is mmc drive
2283                 None          When internal device is something else or
2284                               when we are unable to determine the type
2285        """
2286        # The output should be /dev/mmcblk* for SD/eMMC or /dev/sd* for scsi
2287        rootdev_cmd = ' '.join(['. /usr/sbin/write_gpt.sh;',
2288                                '. /usr/share/misc/chromeos-common.sh;',
2289                                'load_base_vars;',
2290                                'get_fixed_dst_drive'])
2291        rootdev = self.run(command=rootdev_cmd, ignore_status=True)
2292        if rootdev.exit_status:
2293            logging.info("Fail to run %s", rootdev_cmd)
2294            return None
2295        rootdev_str = rootdev.stdout.strip()
2296
2297        if not rootdev_str:
2298            return None
2299
2300        rootdev_base = os.path.basename(rootdev_str)
2301
2302        mmc_pattern = '/dev/mmcblk[0-9]'
2303        if re.match(mmc_pattern, rootdev_str):
2304            # Use type to determine if the internal device is eMMC or somthing
2305            # else. We can assume that MMC is always an internal device.
2306            type_cmd = 'cat /sys/block/%s/device/type' % rootdev_base
2307            type = self.run(command=type_cmd, ignore_status=True)
2308            if type.exit_status:
2309                logging.info("Fail to run %s", type_cmd)
2310                return None
2311            type_str = type.stdout.strip()
2312
2313            if type_str == 'MMC':
2314                return 'storage:mmc'
2315
2316        scsi_pattern = '/dev/sd[a-z]+'
2317        if re.match(scsi_pattern, rootdev.stdout):
2318            # Read symlink for /sys/block/sd* to determine if the internal
2319            # device is connected via ata or usb.
2320            link_cmd = 'readlink /sys/block/%s' % rootdev_base
2321            link = self.run(command=link_cmd, ignore_status=True)
2322            if link.exit_status:
2323                logging.info("Fail to run %s", link_cmd)
2324                return None
2325            link_str = link.stdout.strip()
2326            if 'usb' in link_str:
2327                return None
2328
2329            # Read rotation to determine if the internal device is ssd or hdd.
2330            rotate_cmd = str('cat /sys/block/%s/queue/rotational'
2331                              % rootdev_base)
2332            rotate = self.run(command=rotate_cmd, ignore_status=True)
2333            if rotate.exit_status:
2334                logging.info("Fail to run %s", rotate_cmd)
2335                return None
2336            rotate_str = rotate.stdout.strip()
2337
2338            rotate_dict = {'0':'storage:ssd', '1':'storage:hdd'}
2339            return rotate_dict.get(rotate_str)
2340
2341        # All other internal device / error case will always fall here
2342        return None
2343
2344
2345    def get_servo(self):
2346        """Determine if the host has a servo attached.
2347
2348        If the host has a working servo attached, it should have a servo label.
2349
2350        @return: string 'servo' if the host has servo attached. Otherwise,
2351                 returns None.
2352        """
2353        return 'servo' if self._servo_host else None
2354
2355
2356    def get_video_labels(self):
2357        """Run /usr/local/bin/avtest_label_detect to get a list of video labels.
2358
2359        Sample output of avtest_label_detect:
2360        Detected label: hw_video_acc_vp8
2361        Detected label: webcam
2362
2363        @return: A list of labels detected by tool avtest_label_detect.
2364        """
2365        try:
2366            result = self.run('/usr/local/bin/avtest_label_detect').stdout
2367            return re.findall('^Detected label: (\w+)$', result, re.M)
2368        except error.AutoservRunError:
2369            # The tool is not installed.
2370            return []
2371
2372
2373    def is_video_glitch_detection_supported(self):
2374        """ Determine if a board under test is supported for video glitch
2375        detection tests.
2376
2377        @return: 'video_glitch_detection' if board is supported, None otherwise.
2378        """
2379        board = self.get_board().replace(ds_constants.BOARD_PREFIX, '')
2380
2381        if board in video_test_constants.SUPPORTED_BOARDS:
2382            return 'video_glitch_detection'
2383
2384        return None
2385
2386
2387    def get_touch(self):
2388        """
2389        Determine whether board under test has a touchpad or touchscreen.
2390
2391        @return: A list of some combination of 'touchscreen' and 'touchpad',
2392            depending on what is present on the device.
2393
2394        """
2395        labels = []
2396        looking_for = ['touchpad', 'touchscreen']
2397        player = input_playback.InputPlayback()
2398        input_events = self.run('ls /dev/input/event*').stdout.strip().split()
2399        filename = '/tmp/touch_labels'
2400        for event in input_events:
2401            self.run('evtest %s > %s' % (event, filename), timeout=1,
2402                     ignore_timeout=True)
2403            properties = self.run('cat %s' % filename).stdout
2404            input_type = player._determine_input_type(properties)
2405            if input_type in looking_for:
2406                labels.append(input_type)
2407                looking_for.remove(input_type)
2408            if len(looking_for) == 0:
2409                break
2410        self.run('rm %s' % filename)
2411
2412        return labels
2413
2414
2415    def has_internal_display(self):
2416        """Determine if the device under test is equipped with an internal
2417        display.
2418
2419        @return: 'internal_display' if one is present; None otherwise.
2420        """
2421        from autotest_lib.client.cros.graphics import graphics_utils
2422        from autotest_lib.client.common_lib import utils as common_utils
2423
2424        def __system_output(cmd):
2425            return self.run(cmd).stdout
2426
2427        def __read_file(remote_path):
2428            return self.run('cat %s' % remote_path).stdout
2429
2430        # Hijack the necessary client functions so that we can take advantage
2431        # of the client lib here.
2432        # FIXME: find a less hacky way than this
2433        original_system_output = utils.system_output
2434        original_read_file = common_utils.read_file
2435        utils.system_output = __system_output
2436        common_utils.read_file = __read_file
2437        try:
2438            return ('internal_display' if graphics_utils.has_internal_display()
2439                                   else None)
2440        finally:
2441            utils.system_output = original_system_output
2442            common_utils.read_file = original_read_file
2443
2444
2445    def is_boot_from_usb(self):
2446        """Check if DUT is boot from USB.
2447
2448        @return: True if DUT is boot from usb.
2449        """
2450        device = self.run('rootdev -s -d').stdout.strip()
2451        removable = int(self.run('cat /sys/block/%s/removable' %
2452                                 os.path.basename(device)).stdout.strip())
2453        return removable == 1
2454
2455
2456    def read_from_meminfo(self, key):
2457        """Return the memory info from /proc/meminfo
2458
2459        @param key: meminfo requested
2460
2461        @return the memory value as a string
2462
2463        """
2464        meminfo = self.run('grep %s /proc/meminfo' % key).stdout.strip()
2465        logging.debug('%s', meminfo)
2466        return int(re.search(r'\d+', meminfo).group(0))
2467
2468
2469    def get_cpu_arch(self):
2470        """Returns CPU arch of the device.
2471
2472        @return CPU architecture of the DUT.
2473        """
2474        # Add CPUs by following logic in client/bin/utils.py.
2475        if self.run("grep '^flags.*:.* lm .*' /proc/cpuinfo",
2476                ignore_status=True).stdout:
2477            return 'x86_64'
2478        if self.run("grep -Ei 'ARM|CPU implementer' /proc/cpuinfo",
2479                ignore_status=True).stdout:
2480            return 'arm'
2481        return 'i386'
2482
2483
2484    def get_board_type(self):
2485        """
2486        Get the DUT's device type from /etc/lsb-release.
2487        DEVICETYPE can be one of CHROMEBOX, CHROMEBASE, CHROMEBOOK or more.
2488
2489        @return value of DEVICETYPE param from lsb-release.
2490        """
2491        device_type = self.run('grep DEVICETYPE /etc/lsb-release',
2492                               ignore_status=True).stdout
2493        if device_type:
2494            return device_type.split('=')[-1].strip()
2495        return ''
2496
2497
2498    def get_arc_version(self):
2499        """Return ARC version installed on the DUT.
2500
2501        @returns ARC version as string if the CrOS build has ARC, else None.
2502        """
2503        arc_version = self.run('grep CHROMEOS_ARC_VERSION /etc/lsb-release',
2504                               ignore_status=True).stdout
2505        if arc_version:
2506            return arc_version.split('=')[-1].strip()
2507        return None
2508
2509
2510    def get_os_type(self):
2511        return 'cros'
2512
2513
2514    def enable_adb_testing(self):
2515        """Mark this host as an adb tester."""
2516        self.run('touch %s' % constants.ANDROID_TESTER_FILEFLAG)
2517
2518
2519    def get_labels(self):
2520        """Return the detected labels on the host."""
2521        return self.labels.get_labels(self)
2522
2523
2524    def update_labels(self):
2525        """Update the labels on the host."""
2526        self.labels.update_labels(self)
2527