1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import glob 6import logging 7import os 8import re 9import urlparse 10import urllib2 11 12from autotest_lib.client.bin import utils 13from autotest_lib.client.common_lib import error, global_config 14from autotest_lib.client.common_lib.cros import dev_server 15from autotest_lib.server import utils as server_utils 16from chromite.lib import retry_util 17 18try: 19 from chromite.lib import metrics 20except ImportError: 21 metrics = utils.metrics_mock 22 23try: 24 import devserver 25 STATEFUL_UPDATE_PATH = devserver.__path__[0] 26except ImportError: 27 STATEFUL_UPDATE_PATH = '/usr/bin' 28 29# Local stateful update path is relative to the CrOS source directory. 30STATEFUL_UPDATE_SCRIPT = 'stateful_update' 31UPDATER_IDLE = 'UPDATE_STATUS_IDLE' 32UPDATER_NEED_REBOOT = 'UPDATE_STATUS_UPDATED_NEED_REBOOT' 33# A list of update engine client states that occur after an update is triggered. 34UPDATER_PROCESSING_UPDATE = ['UPDATE_STATUS_CHECKING_FORUPDATE', 35 'UPDATE_STATUS_UPDATE_AVAILABLE', 36 'UPDATE_STATUS_DOWNLOADING', 37 'UPDATE_STATUS_FINALIZING'] 38 39class ChromiumOSError(error.InstallError): 40 """Generic error for ChromiumOS-specific exceptions.""" 41 42 43class RootFSUpdateError(ChromiumOSError): 44 """Raised when the RootFS fails to update.""" 45 46 47class StatefulUpdateError(ChromiumOSError): 48 """Raised when the stateful partition fails to update.""" 49 50 51def url_to_version(update_url): 52 """Return the version based on update_url. 53 54 @param update_url: url to the image to update to. 55 56 """ 57 # The Chrome OS version is generally the last element in the URL. The only 58 # exception is delta update URLs, which are rooted under the version; e.g., 59 # http://.../update/.../0.14.755.0/au/0.14.754.0. In this case we want to 60 # strip off the au section of the path before reading the version. 61 return re.sub('/au/.*', '', 62 urlparse.urlparse(update_url).path).split('/')[-1].strip() 63 64 65def url_to_image_name(update_url): 66 """Return the image name based on update_url. 67 68 From a URL like: 69 http://172.22.50.205:8082/update/lumpy-release/R27-3837.0.0 70 return lumpy-release/R27-3837.0.0 71 72 @param update_url: url to the image to update to. 73 @returns a string representing the image name in the update_url. 74 75 """ 76 return '/'.join(urlparse.urlparse(update_url).path.split('/')[-2:]) 77 78 79def _get_devserver_build_from_update_url(update_url): 80 """Get the devserver and build from the update url. 81 82 @param update_url: The url for update. 83 Eg: http://devserver:port/update/build. 84 85 @return: A tuple of (devserver url, build) or None if the update_url 86 doesn't match the expected pattern. 87 88 @raises ValueError: If the update_url doesn't match the expected pattern. 89 @raises ValueError: If no global_config was found, or it doesn't contain an 90 image_url_pattern. 91 """ 92 pattern = global_config.global_config.get_config_value( 93 'CROS', 'image_url_pattern', type=str, default='') 94 if not pattern: 95 raise ValueError('Cannot parse update_url, the global config needs ' 96 'an image_url_pattern.') 97 re_pattern = pattern.replace('%s', '(\S+)') 98 parts = re.search(re_pattern, update_url) 99 if not parts or len(parts.groups()) < 2: 100 raise ValueError('%s is not an update url' % update_url) 101 return parts.groups() 102 103 104def list_image_dir_contents(update_url): 105 """Lists the contents of the devserver for a given build/update_url. 106 107 @param update_url: An update url. Eg: http://devserver:port/update/build. 108 """ 109 if not update_url: 110 logging.warning('Need update_url to list contents of the devserver.') 111 return 112 error_msg = 'Cannot check contents of devserver, update url %s' % update_url 113 try: 114 devserver_url, build = _get_devserver_build_from_update_url(update_url) 115 except ValueError as e: 116 logging.warning('%s: %s', error_msg, e) 117 return 118 devserver = dev_server.ImageServer(devserver_url) 119 try: 120 devserver.list_image_dir(build) 121 # The devserver will retry on URLError to avoid flaky connections, but will 122 # eventually raise the URLError if it persists. All HTTPErrors get 123 # converted to DevServerExceptions. 124 except (dev_server.DevServerException, urllib2.URLError) as e: 125 logging.warning('%s: %s', error_msg, e) 126 127 128# TODO(garnold) This implements shared updater functionality needed for 129# supporting the autoupdate_EndToEnd server-side test. We should probably 130# migrate more of the existing ChromiumOSUpdater functionality to it as we 131# expand non-CrOS support in other tests. 132class BaseUpdater(object): 133 """Platform-agnostic DUT update functionality.""" 134 135 def __init__(self, updater_ctrl_bin, update_url, host): 136 """Initializes the object. 137 138 @param updater_ctrl_bin: Path to update_engine_client. 139 @param update_url: The URL we want the update to use. 140 @param host: A client.common_lib.hosts.Host implementation. 141 """ 142 self.updater_ctrl_bin = updater_ctrl_bin 143 self.update_url = update_url 144 self.host = host 145 146 147 def check_update_status(self): 148 """Returns the current update engine state. 149 150 We use the `update_engine_client -status' command and parse the line 151 indicating the update state, e.g. "CURRENT_OP=UPDATE_STATUS_IDLE". 152 """ 153 update_status = self.host.run(command='%s -status | grep CURRENT_OP' % 154 self.updater_ctrl_bin) 155 return update_status.stdout.strip().split('=')[-1] 156 157 158 def get_last_update_error(self): 159 """Get the last autoupdate error code.""" 160 error_msg = self.host.run( 161 '%s --last_attempt_error' % self.updater_ctrl_bin) 162 error_msg = (error_msg.stdout.strip()).replace('\n', ', ') 163 return error_msg 164 165 166 def _base_update_handler_no_retry(self, run_args): 167 """Base function to handle a remote update ssh call. 168 169 @param run_args: Dictionary of args passed to ssh_host.run function. 170 171 @throws: intercepts and re-throws all exceptions 172 """ 173 try: 174 self.host.run(**run_args) 175 except Exception as e: 176 logging.debug('exception in update handler: %s', e) 177 raise e 178 179 180 def _base_update_handler(self, run_args, err_msg_prefix=None): 181 """Handle a remote update ssh call, possibly with retries. 182 183 @param run_args: Dictionary of args passed to ssh_host.run function. 184 @param err_msg_prefix: Prefix of the exception error message. 185 """ 186 def exception_handler(e): 187 """Examines exceptions and returns True if the update handler 188 should be retried. 189 190 @param e: the exception intercepted by the retry util. 191 """ 192 return (isinstance(e, error.AutoservSSHTimeout) or 193 (isinstance(e, error.GenericHostRunError) and 194 hasattr(e, 'description') and 195 (re.search('ERROR_CODE=37', e.description) or 196 re.search('generic error .255.', e.description)))) 197 198 try: 199 # Try the update twice (arg 2 is max_retry, not including the first 200 # call). Some exceptions may be caught by the retry handler. 201 retry_util.GenericRetry(exception_handler, 1, 202 self._base_update_handler_no_retry, 203 run_args) 204 except Exception as e: 205 message = err_msg_prefix + ': ' + str(e) 206 raise RootFSUpdateError(message) 207 208 209 def _wait_for_update_service(self): 210 """Ensure that the update engine daemon is running, possibly 211 by waiting for it a bit in case the DUT just rebooted and the 212 service hasn't started yet. 213 """ 214 def handler(e): 215 """Retry exception handler. 216 217 Assumes that the error is due to the update service not having 218 started yet. 219 220 @param e: the exception intercepted by the retry util. 221 """ 222 if isinstance(e, error.AutoservRunError): 223 logging.debug('update service check exception: %s\n' 224 'retrying...', e) 225 return True 226 else: 227 return False 228 229 # Retry at most three times, every 5s. 230 status = retry_util.GenericRetry(handler, 3, 231 self.check_update_status, 232 sleep=5) 233 234 # Expect the update engine to be idle. 235 if status != UPDATER_IDLE: 236 raise ChromiumOSError('%s is not in an installable state' % 237 self.host.hostname) 238 239 240 def trigger_update(self): 241 """Triggers a background update. 242 243 @raise RootFSUpdateError or unknown Exception if anything went wrong. 244 """ 245 # If this function is called immediately after reboot (which it is at 246 # this time), there is no guarantee that the update service is up and 247 # running yet, so wait for it. 248 self._wait_for_update_service() 249 250 autoupdate_cmd = ('%s --check_for_update --omaha_url=%s' % 251 (self.updater_ctrl_bin, self.update_url)) 252 run_args = {'command': autoupdate_cmd} 253 err_prefix = 'Failed to trigger an update on %s. ' % self.host.hostname 254 logging.info('Triggering update via: %s', autoupdate_cmd) 255 metric_fields = {'success': False} 256 try: 257 self._base_update_handler(run_args, err_prefix) 258 metric_fields['success'] = True 259 finally: 260 c = metrics.Counter('chromeos/autotest/autoupdater/trigger') 261 metric_fields.update(self._get_metric_fields()) 262 c.increment(fields=metric_fields) 263 264 265 def _get_metric_fields(self): 266 """Return a dict of metric fields. 267 268 This is used for sending autoupdate metrics for this instance. 269 """ 270 build_name = url_to_image_name(self.update_url) 271 try: 272 board, build_type, milestone, _ = server_utils.ParseBuildName( 273 build_name) 274 except server_utils.ParseBuildNameException: 275 logging.warning('Unable to parse build name %s for metrics. ' 276 'Continuing anyway.', build_name) 277 board, build_type, milestone = ('', '', '') 278 return { 279 'dev_server': dev_server.get_hostname(self.update_url), 280 'board': board, 281 'build_type': build_type, 282 'milestone': milestone, 283 } 284 285 286 def _verify_update_completed(self): 287 """Verifies that an update has completed. 288 289 @raise RootFSUpdateError: if verification fails. 290 """ 291 status = self.check_update_status() 292 if status != UPDATER_NEED_REBOOT: 293 error_msg = '' 294 if status == UPDATER_IDLE: 295 error_msg = 'Update error: %s' % self.get_last_update_error() 296 raise RootFSUpdateError('Update did not complete with correct ' 297 'status. Expecting %s, actual %s. %s' % 298 (UPDATER_NEED_REBOOT, status, error_msg)) 299 300 301 def update_image(self): 302 """Updates the device image and verifies success.""" 303 autoupdate_cmd = ('%s --update --omaha_url=%s' % 304 (self.updater_ctrl_bin, self.update_url)) 305 run_args = {'command': autoupdate_cmd, 'timeout': 3600} 306 err_prefix = ('Failed to install device image using payload at %s ' 307 'on %s. ' % (self.update_url, self.host.hostname)) 308 logging.info('Updating image via: %s', autoupdate_cmd) 309 metric_fields = {'success': False} 310 try: 311 self._base_update_handler(run_args, err_prefix) 312 metric_fields['success'] = True 313 finally: 314 c = metrics.Counter('chromeos/autotest/autoupdater/update') 315 metric_fields.update(self._get_metric_fields()) 316 c.increment(fields=metric_fields) 317 318 self._verify_update_completed() 319 320 321class ChromiumOSUpdater(BaseUpdater): 322 """Helper class used to update DUT with image of desired version.""" 323 REMOTE_STATEFUL_UPDATE_PATH = os.path.join( 324 '/usr/local/bin', STATEFUL_UPDATE_SCRIPT) 325 REMOTE_TMP_STATEFUL_UPDATE = os.path.join( 326 '/tmp', STATEFUL_UPDATE_SCRIPT) 327 UPDATER_BIN = '/usr/bin/update_engine_client' 328 UPDATED_MARKER = '/var/run/update_engine_autoupdate_completed' 329 UPDATER_LOGS = ['/var/log/messages', '/var/log/update_engine'] 330 331 KERNEL_A = {'name': 'KERN-A', 'kernel': 2, 'root': 3} 332 KERNEL_B = {'name': 'KERN-B', 'kernel': 4, 'root': 5} 333 # Time to wait for new kernel to be marked successful after 334 # auto update. 335 KERNEL_UPDATE_TIMEOUT = 120 336 337 def __init__(self, update_url, host=None, local_devserver=False): 338 super(ChromiumOSUpdater, self).__init__(self.UPDATER_BIN, update_url, 339 host) 340 self.local_devserver = local_devserver 341 if not local_devserver: 342 self.update_version = url_to_version(update_url) 343 else: 344 self.update_version = None 345 346 347 def reset_update_engine(self): 348 """Resets the host to prepare for a clean update regardless of state.""" 349 self._run('rm -f %s' % self.UPDATED_MARKER) 350 self._run('stop ui || true') 351 self._run('stop update-engine || true') 352 self._run('start update-engine') 353 354 # Wait for update engine to be ready. 355 self._wait_for_update_service() 356 357 358 def _run(self, cmd, *args, **kwargs): 359 """Abbreviated form of self.host.run(...)""" 360 return self.host.run(cmd, *args, **kwargs) 361 362 363 def rootdev(self, options=''): 364 """Returns the stripped output of rootdev <options>. 365 366 @param options: options to run rootdev. 367 368 """ 369 return self._run('rootdev %s' % options).stdout.strip() 370 371 372 def get_kernel_state(self): 373 """Returns the (<active>, <inactive>) kernel state as a pair.""" 374 active_root = int(re.findall('\d+\Z', self.rootdev('-s'))[0]) 375 if active_root == self.KERNEL_A['root']: 376 return self.KERNEL_A, self.KERNEL_B 377 elif active_root == self.KERNEL_B['root']: 378 return self.KERNEL_B, self.KERNEL_A 379 else: 380 raise ChromiumOSError('Encountered unknown root partition: %s' % 381 active_root) 382 383 384 def _cgpt(self, flag, kernel, dev='$(rootdev -s -d)'): 385 """Return numeric cgpt value for the specified flag, kernel, device. """ 386 return int(self._run('cgpt show -n -i %d %s %s' % ( 387 kernel['kernel'], flag, dev)).stdout.strip()) 388 389 390 def get_kernel_priority(self, kernel): 391 """Return numeric priority for the specified kernel. 392 393 @param kernel: information of the given kernel, KERNEL_A or KERNEL_B. 394 395 """ 396 return self._cgpt('-P', kernel) 397 398 399 def get_kernel_success(self, kernel): 400 """Return boolean success flag for the specified kernel. 401 402 @param kernel: information of the given kernel, KERNEL_A or KERNEL_B. 403 404 """ 405 return self._cgpt('-S', kernel) != 0 406 407 408 def get_kernel_tries(self, kernel): 409 """Return tries count for the specified kernel. 410 411 @param kernel: information of the given kernel, KERNEL_A or KERNEL_B. 412 413 """ 414 return self._cgpt('-T', kernel) 415 416 417 def get_stateful_update_script(self): 418 """Returns the path to the stateful update script on the target. 419 420 When runnning test_that, stateful_update is in chroot /usr/sbin, 421 as installed by chromeos-base/devserver packages. 422 In the lab, it is installed with the python module devserver, by 423 build_externals.py command. 424 425 If we can find it, we hope it exists already on the DUT, we assert 426 otherwise. 427 """ 428 stateful_update_file = os.path.join(STATEFUL_UPDATE_PATH, 429 STATEFUL_UPDATE_SCRIPT) 430 if os.path.exists(stateful_update_file): 431 self.host.send_file( 432 stateful_update_file, self.REMOTE_TMP_STATEFUL_UPDATE, 433 delete_dest=True) 434 return self.REMOTE_TMP_STATEFUL_UPDATE 435 436 if self.host.path_exists(self.REMOTE_STATEFUL_UPDATE_PATH): 437 logging.warning('Could not chroot %s script, falling back on %s', 438 STATEFUL_UPDATE_SCRIPT, self.REMOTE_STATEFUL_UPDATE_PATH) 439 return self.REMOTE_STATEFUL_UPDATE_PATH 440 else: 441 raise ChromiumOSError('Could not locate %s', 442 STATEFUL_UPDATE_SCRIPT) 443 444 445 def reset_stateful_partition(self): 446 """Clear any pending stateful update request.""" 447 statefuldev_cmd = [self.get_stateful_update_script()] 448 statefuldev_cmd += ['--stateful_change=reset', '2>&1'] 449 self._run(' '.join(statefuldev_cmd)) 450 451 452 def revert_boot_partition(self): 453 """Revert the boot partition.""" 454 part = self.rootdev('-s') 455 logging.warning('Reverting update; Boot partition will be %s', part) 456 return self._run('/postinst %s 2>&1' % part) 457 458 459 def rollback_rootfs(self, powerwash): 460 """Triggers rollback and waits for it to complete. 461 462 @param powerwash: If true, powerwash as part of rollback. 463 464 @raise RootFSUpdateError if anything went wrong. 465 466 """ 467 version = self.host.get_release_version() 468 # Introduced can_rollback in M36 (build 5772). # etc/lsb-release matches 469 # X.Y.Z. This version split just pulls the first part out. 470 try: 471 build_number = int(version.split('.')[0]) 472 except ValueError: 473 logging.error('Could not parse build number.') 474 build_number = 0 475 476 if build_number >= 5772: 477 can_rollback_cmd = '%s --can_rollback' % self.UPDATER_BIN 478 logging.info('Checking for rollback.') 479 try: 480 self._run(can_rollback_cmd) 481 except error.AutoservRunError as e: 482 raise RootFSUpdateError("Rollback isn't possible on %s: %s" % 483 (self.host.hostname, str(e))) 484 485 rollback_cmd = '%s --rollback --follow' % self.UPDATER_BIN 486 if not powerwash: 487 rollback_cmd += ' --nopowerwash' 488 489 logging.info('Performing rollback.') 490 try: 491 self._run(rollback_cmd) 492 except error.AutoservRunError as e: 493 raise RootFSUpdateError('Rollback failed on %s: %s' % 494 (self.host.hostname, str(e))) 495 496 self._verify_update_completed() 497 498 499 # TODO(garnold) This is here for backward compatibility and should be 500 # deprecated once we shift to using update_image() everywhere. 501 def update_rootfs(self): 502 """Run the standard command to force an update.""" 503 return self.update_image() 504 505 506 def update_stateful(self, clobber=True): 507 """Updates the stateful partition. 508 509 @param clobber: If True, a clean stateful installation. 510 """ 511 logging.info('Updating stateful partition...') 512 statefuldev_url = self.update_url.replace('update', 513 'static') 514 515 # Attempt stateful partition update; this must succeed so that the newly 516 # installed host is testable after update. 517 statefuldev_cmd = [self.get_stateful_update_script(), statefuldev_url] 518 if clobber: 519 statefuldev_cmd.append('--stateful_change=clean') 520 521 statefuldev_cmd.append('2>&1') 522 try: 523 self._run(' '.join(statefuldev_cmd), timeout=1200) 524 except error.AutoservRunError: 525 update_error = StatefulUpdateError( 526 'Failed to perform stateful update on %s' % 527 self.host.hostname) 528 raise update_error 529 530 def run_update(self, update_root=True): 531 """Update the DUT with image of specific version. 532 533 @param update_root: True to force a rootfs update. 534 """ 535 booted_version = self.host.get_release_version() 536 if self.update_version: 537 logging.info('Updating from version %s to %s.', 538 booted_version, self.update_version) 539 540 # Check that Dev Server is accepting connections (from autoserv's host). 541 # If we can't talk to it, the machine host probably can't either. 542 auserver_host = 'http://%s' % urlparse.urlparse(self.update_url)[1] 543 try: 544 if not dev_server.ImageServer.devserver_healthy(auserver_host): 545 raise ChromiumOSError( 546 'Update server at %s not healthy' % auserver_host) 547 except Exception as e: 548 logging.debug('Error happens in connection to devserver: %r', e) 549 raise ChromiumOSError( 550 'Update server at %s not available' % auserver_host) 551 552 logging.info('Installing from %s to %s', self.update_url, 553 self.host.hostname) 554 555 # Reset update state. 556 self.reset_update_engine() 557 self.reset_stateful_partition() 558 559 try: 560 try: 561 if not update_root: 562 logging.info('Root update is skipped.') 563 else: 564 self.update_rootfs() 565 566 self.update_stateful() 567 except: 568 self.revert_boot_partition() 569 self.reset_stateful_partition() 570 raise 571 572 logging.info('Update complete.') 573 except: 574 # Collect update engine logs in the event of failure. 575 if self.host.job: 576 logging.info('Collecting update engine logs due to failure...') 577 self.host.get_file( 578 self.UPDATER_LOGS, self.host.job.sysinfo.sysinfodir, 579 preserve_perm=False) 580 list_image_dir_contents(self.update_url) 581 raise 582 finally: 583 logging.info('Update engine log has downloaded in ' 584 'sysinfo/update_engine dir. Check the lastest.') 585 586 587 def check_version(self): 588 """Check the image running in DUT has the desired version. 589 590 @returns: True if the DUT's image version matches the version that 591 the autoupdater tries to update to. 592 593 """ 594 booted_version = self.host.get_release_version() 595 return (self.update_version and 596 self.update_version.endswith(booted_version)) 597 598 599 def check_version_to_confirm_install(self): 600 """Check image running in DUT has the desired version to be installed. 601 602 The method should not be used to check if DUT needs to have a full 603 reimage. Only use it to confirm a image is installed. 604 605 The method is designed to verify version for following 6 scenarios with 606 samples of version to update to and expected booted version: 607 1. trybot paladin build. 608 update version: trybot-lumpy-paladin/R27-3837.0.0-b123 609 booted version: 3837.0.2013_03_21_1340 610 611 2. trybot release build. 612 update version: trybot-lumpy-release/R27-3837.0.0-b456 613 booted version: 3837.0.0 614 615 3. buildbot official release build. 616 update version: lumpy-release/R27-3837.0.0 617 booted version: 3837.0.0 618 619 4. non-official paladin rc build. 620 update version: lumpy-paladin/R27-3878.0.0-rc7 621 booted version: 3837.0.0-rc7 622 623 5. chrome-perf build. 624 update version: lumpy-chrome-perf/R28-3837.0.0-b2996 625 booted version: 3837.0.0 626 627 6. pgo-generate build. 628 update version: lumpy-release-pgo-generate/R28-3837.0.0-b2996 629 booted version: 3837.0.0-pgo-generate 630 631 When we are checking if a DUT needs to do a full install, we should NOT 632 use this method to check if the DUT is running the same version, since 633 it may return false positive for a DUT running trybot paladin build to 634 be updated to another trybot paladin build. 635 636 TODO: This logic has a bug if a trybot paladin build failed to be 637 installed in a DUT running an older trybot paladin build with same 638 platform number, but different build number (-b###). So to conclusively 639 determine if a tryjob paladin build is imaged successfully, we may need 640 to find out the date string from update url. 641 642 @returns: True if the DUT's image version (without the date string if 643 the image is a trybot build), matches the version that the 644 autoupdater is trying to update to. 645 646 """ 647 # In the local_devserver case, we can't know the expected 648 # build, so just pass. 649 if not self.update_version: 650 return True 651 652 # Always try the default check_version method first, this prevents 653 # any backward compatibility issue. 654 if self.check_version(): 655 return True 656 657 return utils.version_match(self.update_version, 658 self.host.get_release_version(), 659 self.update_url) 660 661 662 def verify_boot_expectations(self, expected_kernel_state, rollback_message): 663 """Verifies that we fully booted given expected kernel state. 664 665 This method both verifies that we booted using the correct kernel 666 state and that the OS has marked the kernel as good. 667 668 @param expected_kernel_state: kernel state that we are verifying with 669 i.e. I expect to be booted onto partition 4 etc. See output of 670 get_kernel_state. 671 @param rollback_message: string to raise as a ChromiumOSError 672 if we booted with the wrong partition. 673 674 @raises ChromiumOSError: If we didn't. 675 """ 676 # Figure out the newly active kernel. 677 active_kernel_state = self.get_kernel_state()[0] 678 679 # Check for rollback due to a bad build. 680 if (expected_kernel_state and 681 active_kernel_state != expected_kernel_state): 682 683 # Kernel crash reports should be wiped between test runs, but 684 # may persist from earlier parts of the test, or from problems 685 # with provisioning. 686 # 687 # Kernel crash reports will NOT be present if the crash happened 688 # before encrypted stateful is mounted. 689 # 690 # TODO(dgarrett): Integrate with server/crashcollect.py at some 691 # point. 692 kernel_crashes = glob.glob('/var/spool/crash/kernel.*.kcrash') 693 if kernel_crashes: 694 rollback_message += ': kernel_crash' 695 logging.debug('Found %d kernel crash reports:', 696 len(kernel_crashes)) 697 # The crash names contain timestamps that may be useful: 698 # kernel.20131207.005945.0.kcrash 699 for crash in kernel_crashes: 700 logging.debug(' %s', os.path.basename(crash)) 701 702 # Print out some information to make it easier to debug 703 # the rollback. 704 logging.debug('Dumping partition table.') 705 self._run('cgpt show $(rootdev -s -d)') 706 logging.debug('Dumping crossystem for firmware debugging.') 707 self._run('crossystem --all') 708 raise ChromiumOSError(rollback_message) 709 710 # Make sure chromeos-setgoodkernel runs. 711 try: 712 utils.poll_for_condition( 713 lambda: (self.get_kernel_tries(active_kernel_state) == 0 714 and self.get_kernel_success(active_kernel_state)), 715 exception=ChromiumOSError(), 716 timeout=self.KERNEL_UPDATE_TIMEOUT, sleep_interval=5) 717 except ChromiumOSError: 718 services_status = self._run('status system-services').stdout 719 if services_status != 'system-services start/running\n': 720 event = ('Chrome failed to reach login screen') 721 else: 722 event = ('update-engine failed to call ' 723 'chromeos-setgoodkernel') 724 raise ChromiumOSError( 725 'After update and reboot, %s ' 726 'within %d seconds' % (event, 727 self.KERNEL_UPDATE_TIMEOUT)) 728 729