1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6import fcntl 7import glob 8import logging 9import os 10import random 11import re 12import shutil 13import time 14 15import common 16from autotest_lib.client.bin import test, utils 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.cros import constants, cros_logging 19 20 21class CrashTest(test.test): 22 """ 23 This class deals with running crash tests, which are tests which crash a 24 user-space program (or the whole machine) and generate a core dump. We 25 want to check that the correct crash dump is available and can be 26 retrieved. 27 28 Chromium OS has a crash sender which checks for new crash data and sends 29 it to a server. This crash data is used to track software quality and find 30 bugs. The system crash sender normally is always running, but can be paused 31 by creating _PAUSE_FILE. When crash sender sees this, it pauses operation. 32 33 For testing purposes we sometimes want to run the crash sender manually. 34 In this case we can pass the --ignore_pause_file flag and run the crash 35 sender manually. 36 37 Also for testing we sometimes want to mock out the crash sender, and just 38 have it pretend to succeed or fail. The _MOCK_CRASH_SENDING file is used 39 for this. If it doesn't exist, then the crash sender runs normally. If 40 it exists but is empty, the crash sender will succeed (but actually do 41 nothing). If the file contains something, then the crash sender will fail. 42 43 If the user consents to sending crash tests, then the _CONSENT_FILE will 44 exist in the home directory. This test needs to create this file for the 45 crash sending to work. The metrics daemon caches the consent state for 46 1 second, so we need to sleep for more than that after changing it to be 47 sure it picks up the change. 48 49 Crash reports are rate limited to a certain number of reports each 24 50 hours. If the maximum number has already been sent then reports are held 51 until later. This is administered by a directory _CRASH_SENDER_RATE_DIR 52 which contains one temporary file for each time a report is sent. 53 54 The class provides the ability to push a consent file. This disables 55 consent for this test but allows it to be popped back at later. This 56 makes nested tests easier. If _automatic_consent_saving is True (the 57 default) then consent will be pushed at the start and popped at the end. 58 59 Interesting variables: 60 _log_reader: the log reader used for reading log files 61 _leave_crash_sending: True to enable crash sending on exit from the 62 test, False to disable it. (Default True). 63 _automatic_consent_saving: True to push the consent at the start of 64 the test and pop it afterwards. (Default True). 65 66 Useful places to look for more information are: 67 68 chromeos/src/platform/crash-reporter/crash_sender 69 - sender script which crash crash reporter to create reports, then 70 71 chromeos/src/platform/crash-reporter/ 72 - crash reporter program 73 """ 74 75 76 _CONSENT_FILE = '/home/chronos/Consent To Send Stats' 77 _CORE_PATTERN = '/proc/sys/kernel/core_pattern' 78 _LOCK_CORE_PATTERN = '/proc/sys/kernel/lock_core_pattern' 79 _CRASH_REPORTER_PATH = '/sbin/crash_reporter' 80 _CRASH_SENDER_PATH = '/sbin/crash_sender' 81 _CRASH_SENDER_RATE_DIR = '/var/lib/crash_sender' 82 _CRASH_SENDER_LOCK_PATH = '/run/lock/crash_sender' 83 _CRASH_RUN_STATE_DIR = '/run/crash_reporter' 84 _CRASH_TEST_IN_PROGRESS = _CRASH_RUN_STATE_DIR + '/crash-test-in-progress' 85 _MOCK_CRASH_SENDING = _CRASH_RUN_STATE_DIR + '/mock-crash-sending' 86 _PAUSE_FILE = '/var/lib/crash_sender_paused' 87 _SYSTEM_CRASH_DIR = '/var/spool/crash' 88 _FALLBACK_USER_CRASH_DIR = '/home/chronos/crash' 89 _EARLY_BOOT_CRASH_DIR = '/mnt/stateful_partition/unencrypted/preserve/crash' 90 _USER_CRASH_DIRS = '/home/chronos/u-*/crash' 91 _USER_CRASH_DIR_REGEX = re.compile('/home/chronos/u-([a-f0-9]+)/crash') 92 93 # Matches kDefaultMaxUploadBytes 94 _MAX_CRASH_SIZE = 1024 * 1024 95 96 # Use the same file format as crash does normally: 97 # <basename>.#.#.#.meta 98 _FAKE_TEST_BASENAME = 'fake.1.2.3' 99 100 def _set_system_sending(self, is_enabled): 101 """Sets whether or not the system crash_sender is allowed to run. 102 103 This is done by creating or removing _PAUSE_FILE. 104 105 crash_sender may still be allowed to run if _call_sender_one_crash is 106 called with 'ignore_pause=True'. 107 108 @param is_enabled: True to enable crash_sender, False to disable it. 109 """ 110 if is_enabled: 111 if os.path.exists(self._PAUSE_FILE): 112 os.remove(self._PAUSE_FILE) 113 else: 114 utils.system('touch ' + self._PAUSE_FILE) 115 116 117 def _reset_rate_limiting(self): 118 """Reset the count of crash reports sent today. 119 120 This clears the contents of the rate limiting directory which has 121 the effect of reseting our count of crash reports sent. 122 """ 123 utils.system('rm -rf ' + self._CRASH_SENDER_RATE_DIR) 124 125 126 def _clear_spooled_crashes(self): 127 """Clears system and user crash directories. 128 129 This will remove all crash reports which are waiting to be sent. 130 """ 131 utils.system('rm -rf ' + self._SYSTEM_CRASH_DIR) 132 utils.system('rm -rf ' + self._EARLY_BOOT_CRASH_DIR) 133 utils.system('rm -rf %s %s' % (self._USER_CRASH_DIRS, 134 self._FALLBACK_USER_CRASH_DIR)) 135 136 137 def _kill_running_sender(self): 138 """Kill the the crash_sender process if running.""" 139 utils.system('pkill -9 -e --exact crash_sender', ignore_status=True) 140 141 142 def _set_sending_mock(self, mock_enabled, send_success=True): 143 """Enables / disables mocking of the sending process. 144 145 This uses the _MOCK_CRASH_SENDING file to achieve its aims. See notes 146 at the top. 147 148 @param mock_enabled: If True, mocking is enabled, else it is disabled. 149 @param send_success: If mock_enabled this is True for the mocking to 150 indicate success, False to indicate failure. 151 """ 152 if mock_enabled: 153 if send_success: 154 data = '' 155 else: 156 data = '1' 157 logging.info('Setting sending mock') 158 utils.open_write_close(self._MOCK_CRASH_SENDING, data) 159 else: 160 utils.system('rm -f ' + self._MOCK_CRASH_SENDING) 161 162 163 def _set_consent(self, has_consent): 164 """Sets whether or not we have consent to send crash reports. 165 166 This creates or deletes the _CONSENT_FILE to control whether 167 crash_sender will consider that it has consent to send crash reports. 168 It also copies a policy blob with the proper policy setting. 169 170 @param has_consent: True to indicate consent, False otherwise 171 """ 172 autotest_cros_dir = os.path.join(os.path.dirname(__file__), '..') 173 if has_consent: 174 if os.path.isdir(constants.WHITELIST_DIR): 175 # Create policy file that enables metrics/consent. 176 shutil.copy('%s/mock_metrics_on.policy' % autotest_cros_dir, 177 constants.SIGNED_POLICY_FILE) 178 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 179 constants.OWNER_KEY_FILE) 180 # Create deprecated consent file. This is created *after* the 181 # policy file in order to avoid a race condition where chrome 182 # might remove the consent file if the policy's not set yet. 183 # We create it as a temp file first in order to make the creation 184 # of the consent file, owned by chronos, atomic. 185 # See crosbug.com/18413. 186 temp_file = self._CONSENT_FILE + '.tmp'; 187 utils.open_write_close(temp_file, 'test-consent') 188 utils.system('chown chronos:chronos "%s"' % (temp_file)) 189 shutil.move(temp_file, self._CONSENT_FILE) 190 logging.info('Created %s', self._CONSENT_FILE) 191 else: 192 if os.path.isdir(constants.WHITELIST_DIR): 193 # Create policy file that disables metrics/consent. 194 shutil.copy('%s/mock_metrics_off.policy' % autotest_cros_dir, 195 constants.SIGNED_POLICY_FILE) 196 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 197 constants.OWNER_KEY_FILE) 198 # Remove deprecated consent file. 199 utils.system('rm -f "%s"' % (self._CONSENT_FILE)) 200 # Ensure cached consent state is updated. 201 time.sleep(2) 202 203 204 def _set_crash_test_in_progress(self, in_progress): 205 if in_progress: 206 utils.open_write_close(self._CRASH_TEST_IN_PROGRESS, 'in-progress') 207 logging.info('Created %s', self._CRASH_TEST_IN_PROGRESS) 208 else: 209 utils.system('rm -f "%s"' % (self._CRASH_TEST_IN_PROGRESS)) 210 211 212 def _get_pushed_consent_file_path(self): 213 """Returns filename of the pushed consent file.""" 214 return os.path.join(self.bindir, 'pushed_consent') 215 216 217 def _get_pushed_policy_file_path(self): 218 """Returns filename of the pushed policy file.""" 219 return os.path.join(self.bindir, 'pushed_policy') 220 221 222 def _get_pushed_owner_key_file_path(self): 223 """Returns filename of the pushed owner.key file.""" 224 return os.path.join(self.bindir, 'pushed_owner_key') 225 226 227 def _push_consent(self): 228 """Push the consent file, thus disabling consent. 229 230 The consent files can be created in the new test if required. Call 231 _pop_consent() to restore the original state. 232 """ 233 if os.path.exists(self._CONSENT_FILE): 234 shutil.move(self._CONSENT_FILE, 235 self._get_pushed_consent_file_path()) 236 if os.path.exists(constants.SIGNED_POLICY_FILE): 237 shutil.move(constants.SIGNED_POLICY_FILE, 238 self._get_pushed_policy_file_path()) 239 if os.path.exists(constants.OWNER_KEY_FILE): 240 shutil.move(constants.OWNER_KEY_FILE, 241 self._get_pushed_owner_key_file_path()) 242 # Ensure cached consent state is updated. 243 time.sleep(2) 244 245 246 def _pop_consent(self): 247 """Pop the consent files, enabling/disabling consent as it was before 248 we pushed the consent.""" 249 if os.path.exists(self._get_pushed_consent_file_path()): 250 shutil.move(self._get_pushed_consent_file_path(), 251 self._CONSENT_FILE) 252 else: 253 utils.system('rm -f "%s"' % self._CONSENT_FILE) 254 if os.path.exists(self._get_pushed_policy_file_path()): 255 shutil.move(self._get_pushed_policy_file_path(), 256 constants.SIGNED_POLICY_FILE) 257 else: 258 utils.system('rm -f "%s"' % constants.SIGNED_POLICY_FILE) 259 if os.path.exists(self._get_pushed_owner_key_file_path()): 260 shutil.move(self._get_pushed_owner_key_file_path(), 261 constants.OWNER_KEY_FILE) 262 else: 263 utils.system('rm -f "%s"' % constants.OWNER_KEY_FILE) 264 # Ensure cached consent state is updated. 265 time.sleep(2) 266 267 268 def _get_crash_dir(self, username, force_user_crash_dir=False): 269 """Returns crash directory for process running as the given user. 270 271 @param username: Unix user of the crashing process. 272 @param force_user_crash_dir: Regardless of |username|, return the crash 273 directory of the current user session, or 274 the fallback directory if no sessions. 275 """ 276 if username in ('root', 'crash') and not force_user_crash_dir: 277 return self._SYSTEM_CRASH_DIR 278 else: 279 dirs = glob.glob(self._USER_CRASH_DIRS) 280 return dirs[0] if dirs else self._FALLBACK_USER_CRASH_DIR 281 282 283 def _canonicalize_crash_dir(self, crash_dir): 284 """Converts /home/chronos crash directory to /home/user counterpart. 285 286 @param crash_dir: A path of the form /home/chronos/u-<hash>/crash. 287 @returns /home/user/<hash>/crash, or |crash_dir| on form mismatch. 288 """ 289 match = re.match(self._USER_CRASH_DIR_REGEX, crash_dir) 290 return ('/home/user/%s/crash' % match.group(1)) if match else crash_dir 291 292 293 def _initialize_crash_reporter(self, lock_core_pattern): 294 """Start up the crash reporter. 295 296 @param lock_core_pattern: lock core pattern during initialization. 297 """ 298 299 if not lock_core_pattern: 300 self._set_crash_test_in_progress(False) 301 utils.system('%s --init' % self._CRASH_REPORTER_PATH) 302 if not lock_core_pattern: 303 self._set_crash_test_in_progress(True) 304 # Completely disable crash_reporter from generating crash dumps 305 # while any tests are running, otherwise a crashy system can make 306 # these tests flaky. 307 self.enable_crash_filtering('none') 308 309 310 def get_crash_dir_name(self, name): 311 """Return the full path for |name| inside the system crash directory.""" 312 return os.path.join(self._SYSTEM_CRASH_DIR, name) 313 314 315 def write_crash_dir_entry(self, name, contents): 316 """Writes a file to the system crash directory. 317 318 This writes a file to _SYSTEM_CRASH_DIR with the given name. This is 319 used to insert new crash dump files for testing purposes. 320 321 @param name: Name of file to write. 322 @param contents: String to write to the file. 323 """ 324 entry = self.get_crash_dir_name(name) 325 if not os.path.exists(self._SYSTEM_CRASH_DIR): 326 os.makedirs(self._SYSTEM_CRASH_DIR) 327 utils.open_write_close(entry, contents) 328 return entry 329 330 331 def write_fake_meta(self, name, exec_name, payload, complete=True): 332 """Writes a fake meta entry to the system crash directory. 333 334 @param name: Name of file to write. 335 @param exec_name: Value for exec_name item. 336 @param payload: Value for payload item. 337 @param complete: True to close off the record, otherwise leave it 338 incomplete. 339 """ 340 last_line = '' 341 if complete: 342 last_line = 'done=1\n' 343 contents = ('exec_name=%s\n' 344 'ver=my_ver\n' 345 'payload=%s\n' 346 '%s' % (exec_name, payload, 347 last_line)) 348 return self.write_crash_dir_entry(name, contents) 349 350 def _get_dmp_contents(self): 351 """Creates the contents of the dmp file for our made crashes. 352 353 The dmp file contents are deliberately large and hard-to-compress. This 354 ensures logging_CrashSender hits its bytes/day cap before its sends/day 355 cap. 356 """ 357 return bytearray( 358 [random.randint(0, 255) for n in range(self._MAX_CRASH_SIZE)]) 359 360 361 def _prepare_sender_one_crash(self, 362 send_success, 363 reports_enabled, 364 report): 365 """Create metadata for a fake crash report. 366 367 This enabled mocking of the crash sender, then creates a fake 368 crash report for testing purposes. 369 370 @param send_success: True to make the crash_sender success, False to 371 make it fail. 372 @param reports_enabled: True to enable consent to that reports will be 373 sent. 374 @param report: Report to use for crash, if None we create one. 375 """ 376 self._set_sending_mock(mock_enabled=True, send_success=send_success) 377 self._set_consent(reports_enabled) 378 if report is None: 379 # Use the same file format as crash does normally: 380 # <basename>.#.#.#.meta 381 payload = self.write_crash_dir_entry( 382 '%s.dmp' % self._FAKE_TEST_BASENAME, self._get_dmp_contents()) 383 report = self.write_fake_meta( 384 '%s.meta' % self._FAKE_TEST_BASENAME, 'fake', payload) 385 return report 386 387 388 def _parse_sender_output(self, output): 389 """Parse the log output from the crash_sender script. 390 391 This script can run on the logs from either a mocked or true 392 crash send. It looks for one and only one crash from output. 393 Non-crash anomalies should be ignored since there're just noise 394 during running the test. 395 396 @param output: output from the script 397 398 @returns A dictionary with these values: 399 exec_name: name of executable which crashed 400 image_type: type of image ("dev","test",...), if given 401 boot_mode: current boot mode ("dev",...), if given 402 meta_path: path to the report metadata file 403 output: the output from the script, copied 404 report_kind: kind of report sent (minidump vs kernel) 405 send_attempt: did the script attempt to send a crash. 406 send_success: if it attempted, was the crash send successful. 407 sig: signature of the report, if given. 408 sleep_time: if it attempted, how long did it sleep before 409 sending (if mocked, how long would it have slept) 410 """ 411 anomaly_types = ( 412 'kernel_suspend_warning', 413 'kernel_warning', 414 'kernel_wifi_warning', 415 'selinux_violation', 416 'service_failure', 417 ) 418 419 def crash_sender_search(regexp, output): 420 """Narrow search to lines from crash_sender.""" 421 return re.search(r'crash_sender\[\d+\]:\s+' + regexp, output) 422 423 before_first_crash = None 424 while True: 425 crash_header = crash_sender_search( 426 'Considering metadata (\S+)', 427 output 428 ) 429 if not crash_header: 430 break 431 if before_first_crash is None: 432 before_first_crash = output[:crash_header.start()] 433 meta_considered = crash_header.group(1) 434 is_anomaly = any(x in meta_considered for x in anomaly_types) 435 if is_anomaly: 436 # If it's an anomaly, skip this header, and look for next 437 # one. 438 output = output[crash_header.end():] 439 else: 440 # If it's not an anomaly, skip everything before this 441 # header. 442 output = output[crash_header.start():] 443 break 444 if before_first_crash: 445 output = before_first_crash + output 446 logging.debug('Filtered sender output to parse:\n%s', output) 447 448 sleep_match = crash_sender_search('Scheduled to send in (\d+)s', output) 449 send_attempt = sleep_match is not None 450 if send_attempt: 451 sleep_time = int(sleep_match.group(1)) 452 else: 453 sleep_time = None 454 455 meta_match = crash_sender_search('Metadata: (\S+) \((\S+)\)', output) 456 if meta_match: 457 meta_path = meta_match.group(1) 458 report_kind = meta_match.group(2) 459 else: 460 meta_path = None 461 report_kind = None 462 463 payload_match = crash_sender_search('Payload: (\S+)', output) 464 if payload_match: 465 report_payload = payload_match.group(1) 466 else: 467 report_payload = None 468 469 exec_name_match = crash_sender_search('Exec name: (\S+)', output) 470 if exec_name_match: 471 exec_name = exec_name_match.group(1) 472 else: 473 exec_name = None 474 475 sig_match = crash_sender_search('sig: (\S+)', output) 476 if sig_match: 477 sig = sig_match.group(1) 478 else: 479 sig = None 480 481 image_type_match = crash_sender_search('Image type: (\S+)', output) 482 if image_type_match: 483 image_type = image_type_match.group(1) 484 else: 485 image_type = None 486 487 boot_mode_match = crash_sender_search('Boot mode: (\S+)', output) 488 if boot_mode_match: 489 boot_mode = boot_mode_match.group(1) 490 else: 491 boot_mode = None 492 493 send_success = 'Mocking successful send' in output 494 return {'exec_name': exec_name, 495 'report_kind': report_kind, 496 'meta_path': meta_path, 497 'report_payload': report_payload, 498 'send_attempt': send_attempt, 499 'send_success': send_success, 500 'sig': sig, 501 'image_type': image_type, 502 'boot_mode': boot_mode, 503 'sleep_time': sleep_time, 504 'output': output} 505 506 507 def wait_for_sender_completion(self): 508 """Wait for crash_sender to complete. 509 510 Wait for no crash_sender's last message to be placed in the 511 system log before continuing and for the process to finish. 512 Otherwise we might get only part of the output.""" 513 utils.poll_for_condition( 514 lambda: self._log_reader.can_find('crash_sender done.'), 515 timeout=60, 516 exception=error.TestError( 517 'Timeout waiting for crash_sender to emit done: ' + 518 self._log_reader.get_logs())) 519 utils.poll_for_condition( 520 lambda: utils.system('pgrep crash_sender', 521 ignore_status=True) != 0, 522 timeout=60, 523 exception=error.TestError( 524 'Timeout waiting for crash_sender to finish: ' + 525 self._log_reader.get_logs())) 526 527 528 def _call_sender_one_crash(self, 529 send_success=True, 530 reports_enabled=True, 531 report=None, 532 should_fail=False, 533 ignore_pause=True): 534 """Call the crash sender script to mock upload one crash. 535 536 @param send_success: Mock a successful send if true 537 @param reports_enabled: Has the user consented to sending crash reports. 538 @param report: report to use for crash, if None we create one. 539 @param should_fail: expect the crash_sender program to fail 540 @param ignore_pause: crash_sender should ignore pause file existence 541 542 @returns a dictionary describing the result with the keys 543 from _parse_sender_output, as well as: 544 report_exists: does the minidump still exist after calling 545 send script 546 rate_count: how many crashes have been uploaded in the past 547 24 hours. 548 """ 549 report = self._prepare_sender_one_crash(send_success, 550 reports_enabled, 551 report) 552 self._log_reader.set_start_by_current() 553 script_output = "" 554 try: 555 script_output = utils.system_output( 556 '%s %s2>&1' % (self._CRASH_SENDER_PATH, 557 "--ignore_pause_file " if ignore_pause else ""), 558 ignore_status=should_fail) 559 except error.CmdError as err: 560 raise error.TestFail('"%s" returned an unexpected non-zero ' 561 'value (%s).' 562 % (err.command, err.result_obj.exit_status)) 563 564 self.wait_for_sender_completion() 565 output = self._log_reader.get_logs() 566 logging.debug('Crash sender message output:\n %s', output) 567 568 if script_output != '': 569 logging.debug('crash_sender stdout/stderr: %s', script_output) 570 571 if os.path.exists(report): 572 report_exists = True 573 os.remove(report) 574 else: 575 report_exists = False 576 if os.path.exists(self._CRASH_SENDER_RATE_DIR): 577 rate_count = len([ 578 name for name in os.listdir(self._CRASH_SENDER_RATE_DIR) 579 if os.path.isfile(os.path.join(self._CRASH_SENDER_RATE_DIR, 580 name)) 581 ]) 582 else: 583 rate_count = 0 584 585 result = self._parse_sender_output(output) 586 result['report_exists'] = report_exists 587 result['rate_count'] = rate_count 588 589 # Show the result for debugging but remove 'output' key 590 # since it's large and earlier in debug output. 591 debug_result = dict(result) 592 del debug_result['output'] 593 logging.debug('Result of send (besides output): %s', debug_result) 594 595 return result 596 597 598 def _replace_crash_reporter_filter_in(self, new_parameter): 599 """Replaces the --filter_in= parameter of the crash reporter. 600 601 The kernel is set up to call the crash reporter with the core dump 602 as stdin when a process dies. This function adds a filter to the 603 command line used to call the crash reporter. This is used to ignore 604 crashes in which we have no interest. 605 606 This removes any --filter_in= parameter and optionally replaces it 607 with a new one. 608 609 @param new_parameter: This is parameter to add to the command line 610 instead of the --filter_in=... that was there. 611 """ 612 core_pattern = utils.read_file(self._CORE_PATTERN)[:-1] 613 core_pattern = re.sub('--filter_in=\S*\s*', '', 614 core_pattern).rstrip() 615 if new_parameter: 616 core_pattern += ' ' + new_parameter 617 utils.system('echo "%s" > %s' % (core_pattern, self._CORE_PATTERN)) 618 619 620 def enable_crash_filtering(self, name): 621 """Add a --filter_in argument to the kernel core dump cmdline. 622 623 @param name: Filter text to use. This is passed as a --filter_in 624 argument to the crash reporter. 625 """ 626 self._replace_crash_reporter_filter_in('--filter_in=' + name) 627 628 629 def disable_crash_filtering(self): 630 """Remove the --filter_in argument from the kernel core dump cmdline. 631 632 Next time the crash reporter is invoked (due to a crash) it will not 633 receive a --filter_in paramter.""" 634 self._replace_crash_reporter_filter_in('') 635 636 637 @contextlib.contextmanager 638 def hold_crash_lock(self): 639 """A context manager to hold the crash sender lock.""" 640 with open(self._CRASH_SENDER_LOCK_PATH, 'w+') as f: 641 fcntl.lockf(f.fileno(), fcntl.LOCK_EX) 642 try: 643 yield 644 finally: 645 fcntl.lockf(f.fileno(), fcntl.LOCK_UN) 646 647 648 def initialize(self): 649 """Initalize the test.""" 650 test.test.initialize(self) 651 self._log_reader = cros_logging.make_system_log_reader() 652 self._leave_crash_sending = True 653 self._automatic_consent_saving = True 654 self.enable_crash_filtering('none') 655 self._set_crash_test_in_progress(True) 656 657 658 def cleanup(self): 659 """Cleanup after the test. 660 661 We reset things back to the way we think they should be. This is 662 intended to allow the system to continue normal operation. 663 664 Some variables silently change the behavior: 665 _automatic_consent_saving: if True, we pop the consent file. 666 _leave_crash_sending: True to enable crash sending, False to 667 disable it 668 """ 669 self._reset_rate_limiting() 670 self._clear_spooled_crashes() 671 self._set_system_sending(self._leave_crash_sending) 672 self._set_sending_mock(mock_enabled=False) 673 if self._automatic_consent_saving: 674 self._pop_consent() 675 self._set_crash_test_in_progress(False) 676 test.test.cleanup(self) 677 678 679 def run_crash_tests(self, 680 test_names, 681 initialize_crash_reporter=False, 682 clear_spool_first=True, 683 must_run_all=True, 684 lock_core_pattern=False): 685 """Run crash tests defined in this class. 686 687 @param test_names: Array of test names. 688 @param initialize_crash_reporter: Should set up crash reporter for every 689 run. 690 @param clear_spool_first: Clear all spooled user/system crashes before 691 starting the test. 692 @param must_run_all: Should make sure every test in this class is 693 mentioned in test_names. 694 @param lock_core_pattern: Lock core_pattern while initializing 695 crash_reporter. 696 """ 697 if self._automatic_consent_saving: 698 self._push_consent() 699 700 if must_run_all: 701 # Sanity check test_names is complete 702 for attr in dir(self): 703 if attr.find('_test_') == 0: 704 test_name = attr[6:] 705 if not test_name in test_names: 706 raise error.TestError('Test %s is missing' % test_name) 707 708 for test_name in test_names: 709 logging.info(('=' * 20) + ('Running %s' % test_name) + ('=' * 20)) 710 if initialize_crash_reporter: 711 self._initialize_crash_reporter(lock_core_pattern) 712 # Disable crash_sender from running, kill off any running ones. 713 # We set a flag to crash_sender when invoking it manually to avoid 714 # our invocations being paused. 715 self._set_system_sending(False) 716 self._kill_running_sender() 717 self._reset_rate_limiting() 718 if clear_spool_first: 719 self._clear_spooled_crashes() 720 721 # Call the test function 722 getattr(self, '_test_' + test_name)() 723 724 # Clear the intentional crashes, so that the server won't automatically 725 # report crash as failure. 726 self._clear_spooled_crashes() 727