1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6import fcntl 7import glob 8import logging 9import os 10import re 11import shutil 12 13import common 14from autotest_lib.client.bin import test, utils 15from autotest_lib.client.common_lib import error 16from autotest_lib.client.cros import constants, cros_logging 17 18 19class CrashTest(test.test): 20 """ 21 This class deals with running crash tests, which are tests which crash a 22 user-space program (or the whole machine) and generate a core dump. We 23 want to check that the correct crash dump is available and can be 24 retrieved. 25 26 Chromium OS has a crash sender which checks for new crash data and sends 27 it to a server. This crash data is used to track software quality and find 28 bugs. The system crash sender normally is always running, but can be paused 29 by creating _PAUSE_FILE. When crash sender sees this, it pauses operation. 30 31 The pid of the system crash sender is stored in _CRASH_SENDER_RUN_PATH so 32 we can use this to kill the system crash sender for when we want to run 33 our own. 34 35 For testing purposes we sometimes want to run the crash sender manually. 36 In this case we can set 'OVERRIDE_PAUSE_SENDING=1' in the environment and 37 run the crash sender manually (as a child process). 38 39 Also for testing we sometimes want to mock out the crash sender, and just 40 have it pretend to succeed or fail. The _MOCK_CRASH_SENDING file is used 41 for this. If it doesn't exist, then the crash sender runs normally. If 42 it exists but is empty, the crash sender will succeed (but actually do 43 nothing). If the file contains something, then the crash sender will fail. 44 45 If the user consents to sending crash tests, then the _CONSENT_FILE will 46 exist in the home directory. This test needs to create this file for the 47 crash sending to work. 48 49 Crash reports are rate limited to a certain number of reports each 24 50 hours. If the maximum number has already been sent then reports are held 51 until later. This is administered by a directory _CRASH_SENDER_RATE_DIR 52 which contains one temporary file for each time a report is sent. 53 54 The class provides the ability to push a consent file. This disables 55 consent for this test but allows it to be popped back at later. This 56 makes nested tests easier. If _automatic_consent_saving is True (the 57 default) then consent will be pushed at the start and popped at the end. 58 59 Interesting variables: 60 _log_reader: the log reader used for reading log files 61 _leave_crash_sending: True to enable crash sending on exit from the 62 test, False to disable it. (Default True). 63 _automatic_consent_saving: True to push the consent at the start of 64 the test and pop it afterwards. (Default True). 65 66 Useful places to look for more information are: 67 68 chromeos/src/platform/crash-reporter/crash_sender 69 - sender script which crash crash reporter to create reports, then 70 71 chromeos/src/platform/crash-reporter/ 72 - crash reporter program 73 """ 74 75 76 _CONSENT_FILE = '/home/chronos/Consent To Send Stats' 77 _CORE_PATTERN = '/proc/sys/kernel/core_pattern' 78 _CRASH_REPORTER_PATH = '/sbin/crash_reporter' 79 _CRASH_SENDER_PATH = '/sbin/crash_sender' 80 _CRASH_SENDER_RATE_DIR = '/var/lib/crash_sender' 81 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' 82 _CRASH_SENDER_LOCK_PATH = '/var/lock/crash_sender' 83 _CRASH_TEST_IN_PROGRESS = '/tmp/crash-test-in-progress' 84 _MOCK_CRASH_SENDING = '/tmp/mock-crash-sending' 85 _PAUSE_FILE = '/var/lib/crash_sender_paused' 86 _SYSTEM_CRASH_DIR = '/var/spool/crash' 87 _FALLBACK_USER_CRASH_DIR = '/home/chronos/crash' 88 _USER_CRASH_DIRS = '/home/chronos/u-*/crash' 89 _USER_CRASH_DIR_REGEX = re.compile('/home/chronos/u-([a-f0-9]+)/crash') 90 91 # Use the same file format as crash does normally: 92 # <basename>.#.#.#.meta 93 _FAKE_TEST_BASENAME = 'fake.1.2.3' 94 95 def _set_system_sending(self, is_enabled): 96 """Sets whether or not the system crash_sender is allowed to run. 97 98 This is done by creating or removing _PAUSE_FILE. 99 100 crash_sender may still be allowed to run if _set_child_sending is 101 called with True and it is run as a child process. 102 103 @param is_enabled: True to enable crash_sender, False to disable it. 104 """ 105 if is_enabled: 106 if os.path.exists(self._PAUSE_FILE): 107 os.remove(self._PAUSE_FILE) 108 else: 109 utils.system('touch ' + self._PAUSE_FILE) 110 111 112 def _set_child_sending(self, is_enabled): 113 """Overrides crash sending enabling for child processes. 114 115 When the system crash sender is disabled this test can manually run 116 the crash sender as a child process. Normally this would do nothing, 117 but this function sets up crash_sender to ignore its disabled status 118 and do its job. 119 120 @param is_enabled: True to enable crash sending for child processes. 121 """ 122 if is_enabled: 123 os.environ['OVERRIDE_PAUSE_SENDING'] = "1" 124 else: 125 del os.environ['OVERRIDE_PAUSE_SENDING'] 126 127 128 def _set_force_official(self, is_enabled): 129 """Sets whether or not reports will upload for unofficial versions. 130 131 Normally, crash reports are only uploaded for official build 132 versions. If the override is set, however, they will also be 133 uploaded for unofficial versions. 134 135 @param is_enabled: True to enable uploading for unofficial versions. 136 """ 137 if is_enabled: 138 os.environ['FORCE_OFFICIAL'] = "1" 139 elif os.environ.get('FORCE_OFFICIAL'): 140 del os.environ['FORCE_OFFICIAL'] 141 142 143 def _set_mock_developer_mode(self, is_enabled): 144 """Sets whether or not we should pretend we booted in developer mode. 145 146 @param is_enabled: True to pretend we are in developer mode. 147 """ 148 if is_enabled: 149 os.environ['MOCK_DEVELOPER_MODE'] = "1" 150 elif os.environ.get('MOCK_DEVELOPER_MODE'): 151 del os.environ['MOCK_DEVELOPER_MODE'] 152 153 154 def _reset_rate_limiting(self): 155 """Reset the count of crash reports sent today. 156 157 This clears the contents of the rate limiting directory which has 158 the effect of reseting our count of crash reports sent. 159 """ 160 utils.system('rm -rf ' + self._CRASH_SENDER_RATE_DIR) 161 162 163 def _clear_spooled_crashes(self): 164 """Clears system and user crash directories. 165 166 This will remove all crash reports which are waiting to be sent. 167 """ 168 utils.system('rm -rf ' + self._SYSTEM_CRASH_DIR) 169 utils.system('rm -rf %s %s' % (self._USER_CRASH_DIRS, 170 self._FALLBACK_USER_CRASH_DIR)) 171 172 173 def _kill_running_sender(self): 174 """Kill the the crash_sender process if running. 175 176 We use the PID file to find the process ID, then kill it with signal 9. 177 """ 178 if not os.path.exists(self._CRASH_SENDER_RUN_PATH): 179 return 180 running_pid = int(utils.read_file(self._CRASH_SENDER_RUN_PATH)) 181 logging.warning('Detected running crash sender (%d), killing', 182 running_pid) 183 utils.system('kill -9 %d' % running_pid) 184 os.remove(self._CRASH_SENDER_RUN_PATH) 185 186 187 def _set_sending_mock(self, mock_enabled, send_success=True): 188 """Enables / disables mocking of the sending process. 189 190 This uses the _MOCK_CRASH_SENDING file to achieve its aims. See notes 191 at the top. 192 193 @param mock_enabled: If True, mocking is enabled, else it is disabled. 194 @param send_success: If mock_enabled this is True for the mocking to 195 indicate success, False to indicate failure. 196 """ 197 if mock_enabled: 198 if send_success: 199 data = '' 200 else: 201 data = '1' 202 logging.info('Setting sending mock') 203 utils.open_write_close(self._MOCK_CRASH_SENDING, data) 204 else: 205 utils.system('rm -f ' + self._MOCK_CRASH_SENDING) 206 207 208 def _set_consent(self, has_consent): 209 """Sets whether or not we have consent to send crash reports. 210 211 This creates or deletes the _CONSENT_FILE to control whether 212 crash_sender will consider that it has consent to send crash reports. 213 It also copies a policy blob with the proper policy setting. 214 215 @param has_consent: True to indicate consent, False otherwise 216 """ 217 autotest_cros_dir = os.path.join(os.path.dirname(__file__), '..') 218 if has_consent: 219 if os.path.isdir(constants.WHITELIST_DIR): 220 # Create policy file that enables metrics/consent. 221 shutil.copy('%s/mock_metrics_on.policy' % autotest_cros_dir, 222 constants.SIGNED_POLICY_FILE) 223 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 224 constants.OWNER_KEY_FILE) 225 # Create deprecated consent file. This is created *after* the 226 # policy file in order to avoid a race condition where chrome 227 # might remove the consent file if the policy's not set yet. 228 # We create it as a temp file first in order to make the creation 229 # of the consent file, owned by chronos, atomic. 230 # See crosbug.com/18413. 231 temp_file = self._CONSENT_FILE + '.tmp'; 232 utils.open_write_close(temp_file, 'test-consent') 233 utils.system('chown chronos:chronos "%s"' % (temp_file)) 234 shutil.move(temp_file, self._CONSENT_FILE) 235 logging.info('Created ' + self._CONSENT_FILE) 236 else: 237 if os.path.isdir(constants.WHITELIST_DIR): 238 # Create policy file that disables metrics/consent. 239 shutil.copy('%s/mock_metrics_off.policy' % autotest_cros_dir, 240 constants.SIGNED_POLICY_FILE) 241 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 242 constants.OWNER_KEY_FILE) 243 # Remove deprecated consent file. 244 utils.system('rm -f "%s"' % (self._CONSENT_FILE)) 245 246 247 def _set_crash_test_in_progress(self, in_progress): 248 if in_progress: 249 utils.open_write_close(self._CRASH_TEST_IN_PROGRESS, 'in-progress') 250 logging.info('Created ' + self._CRASH_TEST_IN_PROGRESS) 251 else: 252 utils.system('rm -f "%s"' % (self._CRASH_TEST_IN_PROGRESS)) 253 254 255 def _get_pushed_consent_file_path(self): 256 """Returns filename of the pushed consent file.""" 257 return os.path.join(self.bindir, 'pushed_consent') 258 259 260 def _get_pushed_policy_file_path(self): 261 """Returns filename of the pushed policy file.""" 262 return os.path.join(self.bindir, 'pushed_policy') 263 264 265 def _get_pushed_owner_key_file_path(self): 266 """Returns filename of the pushed owner.key file.""" 267 return os.path.join(self.bindir, 'pushed_owner_key') 268 269 270 def _push_consent(self): 271 """Push the consent file, thus disabling consent. 272 273 The consent files can be created in the new test if required. Call 274 _pop_consent() to restore the original state. 275 """ 276 if os.path.exists(self._CONSENT_FILE): 277 shutil.move(self._CONSENT_FILE, 278 self._get_pushed_consent_file_path()) 279 if os.path.exists(constants.SIGNED_POLICY_FILE): 280 shutil.move(constants.SIGNED_POLICY_FILE, 281 self._get_pushed_policy_file_path()) 282 if os.path.exists(constants.OWNER_KEY_FILE): 283 shutil.move(constants.OWNER_KEY_FILE, 284 self._get_pushed_owner_key_file_path()) 285 286 287 def _pop_consent(self): 288 """Pop the consent files, enabling/disabling consent as it was before 289 we pushed the consent.""" 290 if os.path.exists(self._get_pushed_consent_file_path()): 291 shutil.move(self._get_pushed_consent_file_path(), 292 self._CONSENT_FILE) 293 else: 294 utils.system('rm -f "%s"' % self._CONSENT_FILE) 295 if os.path.exists(self._get_pushed_policy_file_path()): 296 shutil.move(self._get_pushed_policy_file_path(), 297 constants.SIGNED_POLICY_FILE) 298 else: 299 utils.system('rm -f "%s"' % constants.SIGNED_POLICY_FILE) 300 if os.path.exists(self._get_pushed_owner_key_file_path()): 301 shutil.move(self._get_pushed_owner_key_file_path(), 302 constants.OWNER_KEY_FILE) 303 else: 304 utils.system('rm -f "%s"' % constants.OWNER_KEY_FILE) 305 306 307 def _get_crash_dir(self, username, force_user_crash_dir=False): 308 """Returns crash directory for process running as the given user. 309 310 @param username: Unix user of the crashing process. 311 @param force_user_crash_dir: Regardless of |username|, return the crash 312 directory of the current user session, or 313 the fallback directory if no sessions. 314 """ 315 if username == 'root' and not force_user_crash_dir: 316 return self._SYSTEM_CRASH_DIR 317 else: 318 dirs = glob.glob(self._USER_CRASH_DIRS) 319 return dirs[0] if dirs else self._FALLBACK_USER_CRASH_DIR 320 321 322 def _canonicalize_crash_dir(self, crash_dir): 323 """Converts /home/chronos crash directory to /home/user counterpart. 324 325 @param crash_dir: A path of the form /home/chronos/u-<hash>/crash. 326 @returns /home/user/<hash>/crash, or |crash_dir| on form mismatch. 327 """ 328 match = re.match(self._USER_CRASH_DIR_REGEX, crash_dir) 329 return ('/home/user/%s/crash' % match.group(1)) if match else crash_dir 330 331 332 def _initialize_crash_reporter(self): 333 """Start up the crash reporter.""" 334 utils.system('%s --init' % self._CRASH_REPORTER_PATH) 335 # Completely disable crash_reporter from generating crash dumps 336 # while any tests are running, otherwise a crashy system can make 337 # these tests flaky. 338 self.enable_crash_filtering('none') 339 340 341 def get_crash_dir_name(self, name): 342 """Return the full path for |name| inside the system crash directory.""" 343 return os.path.join(self._SYSTEM_CRASH_DIR, name) 344 345 346 def write_crash_dir_entry(self, name, contents): 347 """Writes an empty file to the system crash directory. 348 349 This writes a file to _SYSTEM_CRASH_DIR with the given name. This is 350 used to insert new crash dump files for testing purposes. 351 352 @param name: Name of file to write. 353 @param contents: String to write to the file. 354 """ 355 entry = self.get_crash_dir_name(name) 356 if not os.path.exists(self._SYSTEM_CRASH_DIR): 357 os.makedirs(self._SYSTEM_CRASH_DIR) 358 utils.open_write_close(entry, contents) 359 return entry 360 361 362 def write_fake_meta(self, name, exec_name, payload, log=None, 363 complete=True): 364 """Writes a fake meta entry to the system crash directory. 365 366 @param name: Name of file to write. 367 @param exec_name: Value for exec_name item. 368 @param payload: Value for payload item. 369 @param log: Value for log item. 370 @param complete: True to close off the record, otherwise leave it 371 incomplete. 372 """ 373 last_line = '' 374 if complete: 375 last_line = 'done=1\n' 376 contents = ('exec_name=%s\n' 377 'ver=my_ver\n' 378 'payload=%s\n' 379 '%s' % (exec_name, payload, 380 last_line)) 381 if log: 382 contents = ('log=%s\n' % log) + contents 383 return self.write_crash_dir_entry(name, contents) 384 385 386 def _prepare_sender_one_crash(self, 387 send_success, 388 reports_enabled, 389 report): 390 """Create metadata for a fake crash report. 391 392 This enabled mocking of the crash sender, then creates a fake 393 crash report for testing purposes. 394 395 @param send_success: True to make the crash_sender success, False to 396 make it fail. 397 @param reports_enabled: True to enable consent to that reports will be 398 sent. 399 @param report: Report to use for crash, if None we create one. 400 """ 401 self._set_sending_mock(mock_enabled=True, send_success=send_success) 402 self._set_consent(reports_enabled) 403 if report is None: 404 # Use the same file format as crash does normally: 405 # <basename>.#.#.#.meta 406 payload = self.write_crash_dir_entry( 407 '%s.dmp' % self._FAKE_TEST_BASENAME, '') 408 report = self.write_fake_meta( 409 '%s.meta' % self._FAKE_TEST_BASENAME, 'fake', payload) 410 return report 411 412 413 def _parse_sender_output(self, output): 414 """Parse the log output from the crash_sender script. 415 416 This script can run on the logs from either a mocked or true 417 crash send. 418 419 @param output: output from the script 420 421 @returns A dictionary with these values: 422 error_type: an error type, if given 423 exec_name: name of executable which crashed 424 image_type: type of image ("dev","force-official",...), if given 425 boot_mode: current boot mode ("dev",...), if given 426 meta_path: path to the report metadata file 427 output: the output from the script, copied 428 report_kind: kind of report sent (minidump vs kernel) 429 send_attempt: did the script attempt to send a crash. 430 send_success: if it attempted, was the crash send successful. 431 sig: signature of the report, if given. 432 sleep_time: if it attempted, how long did it sleep before 433 sending (if mocked, how long would it have slept) 434 """ 435 sleep_match = re.search('Scheduled to send in (\d+)s', output) 436 send_attempt = sleep_match is not None 437 if send_attempt: 438 sleep_time = int(sleep_match.group(1)) 439 else: 440 sleep_time = None 441 442 meta_match = re.search('Metadata: (\S+) \((\S+)\)', output) 443 if meta_match: 444 meta_path = meta_match.group(1) 445 report_kind = meta_match.group(2) 446 else: 447 meta_path = None 448 report_kind = None 449 450 payload_match = re.search('Payload: (\S+)', output) 451 if payload_match: 452 report_payload = payload_match.group(1) 453 else: 454 report_payload = None 455 456 exec_name_match = re.search('Exec name: (\S+)', output) 457 if exec_name_match: 458 exec_name = exec_name_match.group(1) 459 else: 460 exec_name = None 461 462 sig_match = re.search('sig: (\S+)', output) 463 if sig_match: 464 sig = sig_match.group(1) 465 else: 466 sig = None 467 468 error_type_match = re.search('Error type: (\S+)', output) 469 if error_type_match: 470 error_type = error_type_match.group(1) 471 else: 472 error_type = None 473 474 image_type_match = re.search('Image type: (\S+)', output) 475 if image_type_match: 476 image_type = image_type_match.group(1) 477 else: 478 image_type = None 479 480 boot_mode_match = re.search('Boot mode: (\S+)', output) 481 if boot_mode_match: 482 boot_mode = boot_mode_match.group(1) 483 else: 484 boot_mode = None 485 486 send_success = 'Mocking successful send' in output 487 return {'exec_name': exec_name, 488 'report_kind': report_kind, 489 'meta_path': meta_path, 490 'report_payload': report_payload, 491 'send_attempt': send_attempt, 492 'send_success': send_success, 493 'sig': sig, 494 'error_type': error_type, 495 'image_type': image_type, 496 'boot_mode': boot_mode, 497 'sleep_time': sleep_time, 498 'output': output} 499 500 501 def wait_for_sender_completion(self): 502 """Wait for crash_sender to complete. 503 504 Wait for no crash_sender's last message to be placed in the 505 system log before continuing and for the process to finish. 506 Otherwise we might get only part of the output.""" 507 utils.poll_for_condition( 508 lambda: self._log_reader.can_find('crash_sender done.'), 509 timeout=60, 510 exception=error.TestError( 511 'Timeout waiting for crash_sender to emit done: ' + 512 self._log_reader.get_logs())) 513 utils.poll_for_condition( 514 lambda: utils.system('pgrep crash_sender', 515 ignore_status=True) != 0, 516 timeout=60, 517 exception=error.TestError( 518 'Timeout waiting for crash_sender to finish: ' + 519 self._log_reader.get_logs())) 520 521 522 def _call_sender_one_crash(self, 523 send_success=True, 524 reports_enabled=True, 525 report=None, 526 should_fail=False): 527 """Call the crash sender script to mock upload one crash. 528 529 @param send_success: Mock a successful send if true 530 @param reports_enabled: Has the user consented to sending crash reports. 531 @param report: report to use for crash, if None we create one. 532 533 @returns a dictionary describing the result with the keys 534 from _parse_sender_output, as well as: 535 report_exists: does the minidump still exist after calling 536 send script 537 rate_count: how many crashes have been uploaded in the past 538 24 hours. 539 """ 540 report = self._prepare_sender_one_crash(send_success, 541 reports_enabled, 542 report) 543 self._log_reader.set_start_by_current() 544 script_output = "" 545 try: 546 script_output = utils.system_output( 547 '/bin/sh -c "%s" 2>&1' % self._CRASH_SENDER_PATH, 548 ignore_status=should_fail) 549 except error.CmdError as err: 550 raise error.TestFail('"%s" returned an unexpected non-zero ' 551 'value (%s).' 552 % (err.command, err.result_obj.exit_status)) 553 554 self.wait_for_sender_completion() 555 output = self._log_reader.get_logs() 556 logging.debug('Crash sender message output:\n' + output) 557 558 if script_output != '': 559 logging.debug('crash_sender stdout/stderr: ' + script_output) 560 561 if os.path.exists(report): 562 report_exists = True 563 os.remove(report) 564 else: 565 report_exists = False 566 if os.path.exists(self._CRASH_SENDER_RATE_DIR): 567 rate_count = len(os.listdir(self._CRASH_SENDER_RATE_DIR)) 568 else: 569 rate_count = 0 570 571 result = self._parse_sender_output(output) 572 result['report_exists'] = report_exists 573 result['rate_count'] = rate_count 574 575 # Show the result for debugging but remove 'output' key 576 # since it's large and earlier in debug output. 577 debug_result = dict(result) 578 del debug_result['output'] 579 logging.debug('Result of send (besides output): %s', debug_result) 580 581 return result 582 583 584 def _replace_crash_reporter_filter_in(self, new_parameter): 585 """Replaces the --filter_in= parameter of the crash reporter. 586 587 The kernel is set up to call the crash reporter with the core dump 588 as stdin when a process dies. This function adds a filter to the 589 command line used to call the crash reporter. This is used to ignore 590 crashes in which we have no interest. 591 592 This removes any --filter_in= parameter and optionally replaces it 593 with a new one. 594 595 @param new_parameter: This is parameter to add to the command line 596 instead of the --filter_in=... that was there. 597 """ 598 core_pattern = utils.read_file(self._CORE_PATTERN)[:-1] 599 core_pattern = re.sub('--filter_in=\S*\s*', '', 600 core_pattern).rstrip() 601 if new_parameter: 602 core_pattern += ' ' + new_parameter 603 utils.system('echo "%s" > %s' % (core_pattern, self._CORE_PATTERN)) 604 605 606 def enable_crash_filtering(self, name): 607 """Add a --filter_in argument to the kernel core dump cmdline. 608 609 @param name: Filter text to use. This is passed as a --filter_in 610 argument to the crash reporter. 611 """ 612 self._replace_crash_reporter_filter_in('--filter_in=' + name) 613 614 615 def disable_crash_filtering(self): 616 """Remove the --filter_in argument from the kernel core dump cmdline. 617 618 Next time the crash reporter is invoked (due to a crash) it will not 619 receive a --filter_in paramter.""" 620 self._replace_crash_reporter_filter_in('') 621 622 623 @contextlib.contextmanager 624 def hold_crash_lock(self): 625 """A context manager to hold the crash sender lock.""" 626 with open(self._CRASH_SENDER_LOCK_PATH, 'w+') as f: 627 fcntl.flock(f.fileno(), fcntl.LOCK_EX) 628 try: 629 yield 630 finally: 631 fcntl.flock(f.fileno(), fcntl.LOCK_UN) 632 633 634 def initialize(self): 635 """Initalize the test.""" 636 test.test.initialize(self) 637 self._log_reader = cros_logging.make_system_log_reader() 638 self._leave_crash_sending = True 639 self._automatic_consent_saving = True 640 self.enable_crash_filtering('none') 641 self._set_crash_test_in_progress(True) 642 643 644 def cleanup(self): 645 """Cleanup after the test. 646 647 We reset things back to the way we think they should be. This is 648 intended to allow the system to continue normal operation. 649 650 Some variables silently change the behavior: 651 _automatic_consent_saving: if True, we pop the consent file. 652 _leave_crash_sending: True to enable crash sending, False to 653 disable it 654 """ 655 self._reset_rate_limiting() 656 self._clear_spooled_crashes() 657 self._set_system_sending(self._leave_crash_sending) 658 self._set_sending_mock(mock_enabled=False) 659 if self._automatic_consent_saving: 660 self._pop_consent() 661 self.disable_crash_filtering() 662 self._set_crash_test_in_progress(False) 663 test.test.cleanup(self) 664 665 666 def run_crash_tests(self, 667 test_names, 668 initialize_crash_reporter=False, 669 clear_spool_first=True, 670 must_run_all=True): 671 """Run crash tests defined in this class. 672 673 @param test_names: Array of test names. 674 @param initialize_crash_reporter: Should set up crash reporter for every 675 run. 676 @param clear_spool_first: Clear all spooled user/system crashes before 677 starting the test. 678 @param must_run_all: Should make sure every test in this class is 679 mentioned in test_names. 680 """ 681 if self._automatic_consent_saving: 682 self._push_consent() 683 684 if must_run_all: 685 # Sanity check test_names is complete 686 for attr in dir(self): 687 if attr.find('_test_') == 0: 688 test_name = attr[6:] 689 if not test_name in test_names: 690 raise error.TestError('Test %s is missing' % test_name) 691 692 for test_name in test_names: 693 logging.info(('=' * 20) + ('Running %s' % test_name) + ('=' * 20)) 694 if initialize_crash_reporter: 695 self._initialize_crash_reporter() 696 # Disable crash_sender from running, kill off any running ones, but 697 # set environment so crash_sender may run as a child process. 698 self._set_system_sending(False) 699 self._set_child_sending(True) 700 self._kill_running_sender() 701 self._reset_rate_limiting() 702 # Default to not overriding for unofficial versions. 703 self._set_force_official(False) 704 # Default to not pretending we're in developer mode. 705 self._set_mock_developer_mode(False) 706 if clear_spool_first: 707 self._clear_spooled_crashes() 708 709 # Call the test function 710 getattr(self, '_test_' + test_name)() 711 712 # Clear the intentional crashes, so that the server won't automatically 713 # report crash as failure. 714 self._clear_spooled_crashes() 715