1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib, fcntl, logging, os, re, shutil 6 7import common, constants, cros_logging 8from autotest_lib.client.bin import test, utils 9from autotest_lib.client.common_lib import error 10 11 12class CrashTest(test.test): 13 """ 14 This class deals with running crash tests, which are tests which crash a 15 user-space program (or the whole machine) and generate a core dump. We 16 want to check that the correct crash dump is available and can be 17 retrieved. 18 19 Chromium OS has a crash sender which checks for new crash data and sends 20 it to a server. This crash data is used to track software quality and find 21 bugs. The system crash sender normally is always running, but can be paused 22 by creating _PAUSE_FILE. When crash sender sees this, it pauses operation. 23 24 The pid of the system crash sender is stored in _CRASH_SENDER_RUN_PATH so 25 we can use this to kill the system crash sender for when we want to run 26 our own. 27 28 For testing purposes we sometimes want to run the crash sender manually. 29 In this case we can set 'OVERRIDE_PAUSE_SENDING=1' in the environment and 30 run the crash sender manually (as a child process). 31 32 Also for testing we sometimes want to mock out the crash sender, and just 33 have it pretend to succeed or fail. The _MOCK_CRASH_SENDING file is used 34 for this. If it doesn't exist, then the crash sender runs normally. If 35 it exists but is empty, the crash sender will succeed (but actually do 36 nothing). If the file contains something, then the crash sender will fail. 37 38 If the user consents to sending crash tests, then the _CONSENT_FILE will 39 exist in the home directory. This test needs to create this file for the 40 crash sending to work. 41 42 Crash reports are rate limited to a certain number of reports each 24 43 hours. If the maximum number has already been sent then reports are held 44 until later. This is administered by a directory _CRASH_SENDER_RATE_DIR 45 which contains one temporary file for each time a report is sent. 46 47 The class provides the ability to push a consent file. This disables 48 consent for this test but allows it to be popped back at later. This 49 makes nested tests easier. If _automatic_consent_saving is True (the 50 default) then consent will be pushed at the start and popped at the end. 51 52 Interesting variables: 53 _log_reader: the log reader used for reading log files 54 _leave_crash_sending: True to enable crash sending on exit from the 55 test, False to disable it. (Default True). 56 _automatic_consent_saving: True to push the consent at the start of 57 the test and pop it afterwards. (Default True). 58 59 Useful places to look for more information are: 60 61 chromeos/src/platform/crash-reporter/crash_sender 62 - sender script which crash crash reporter to create reports, then 63 64 chromeos/src/platform/crash-reporter/ 65 - crash reporter program 66 """ 67 68 69 _CONSENT_FILE = '/home/chronos/Consent To Send Stats' 70 _CORE_PATTERN = '/proc/sys/kernel/core_pattern' 71 _CRASH_REPORTER_PATH = '/sbin/crash_reporter' 72 _CRASH_SENDER_PATH = '/sbin/crash_sender' 73 _CRASH_SENDER_RATE_DIR = '/var/lib/crash_sender' 74 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' 75 _CRASH_SENDER_LOCK_PATH = '/var/lock/crash_sender' 76 _CRASH_TEST_IN_PROGRESS = '/tmp/crash-test-in-progress' 77 _MOCK_CRASH_SENDING = '/tmp/mock-crash-sending' 78 _PAUSE_FILE = '/var/lib/crash_sender_paused' 79 _SYSTEM_CRASH_DIR = '/var/spool/crash' 80 _FALLBACK_USER_CRASH_DIR = '/home/chronos/crash' 81 _USER_CRASH_DIRS = '/home/chronos/u-*/crash' 82 83 # Use the same file format as crash does normally: 84 # <basename>.#.#.#.meta 85 _FAKE_TEST_BASENAME = 'fake.1.2.3' 86 87 def _set_system_sending(self, is_enabled): 88 """Sets whether or not the system crash_sender is allowed to run. 89 90 This is done by creating or removing _PAUSE_FILE. 91 92 crash_sender may still be allowed to run if _set_child_sending is 93 called with True and it is run as a child process. 94 95 @param is_enabled: True to enable crash_sender, False to disable it. 96 """ 97 if is_enabled: 98 if os.path.exists(self._PAUSE_FILE): 99 os.remove(self._PAUSE_FILE) 100 else: 101 utils.system('touch ' + self._PAUSE_FILE) 102 103 104 def _set_child_sending(self, is_enabled): 105 """Overrides crash sending enabling for child processes. 106 107 When the system crash sender is disabled this test can manually run 108 the crash sender as a child process. Normally this would do nothing, 109 but this function sets up crash_sender to ignore its disabled status 110 and do its job. 111 112 @param is_enabled: True to enable crash sending for child processes. 113 """ 114 if is_enabled: 115 os.environ['OVERRIDE_PAUSE_SENDING'] = "1" 116 else: 117 del os.environ['OVERRIDE_PAUSE_SENDING'] 118 119 120 def _set_force_official(self, is_enabled): 121 """Sets whether or not reports will upload for unofficial versions. 122 123 Normally, crash reports are only uploaded for official build 124 versions. If the override is set, however, they will also be 125 uploaded for unofficial versions. 126 127 @param is_enabled: True to enable uploading for unofficial versions. 128 """ 129 if is_enabled: 130 os.environ['FORCE_OFFICIAL'] = "1" 131 elif os.environ.get('FORCE_OFFICIAL'): 132 del os.environ['FORCE_OFFICIAL'] 133 134 135 def _set_mock_developer_mode(self, is_enabled): 136 """Sets whether or not we should pretend we booted in developer mode. 137 138 @param is_enabled: True to pretend we are in developer mode. 139 """ 140 if is_enabled: 141 os.environ['MOCK_DEVELOPER_MODE'] = "1" 142 elif os.environ.get('MOCK_DEVELOPER_MODE'): 143 del os.environ['MOCK_DEVELOPER_MODE'] 144 145 146 def _reset_rate_limiting(self): 147 """Reset the count of crash reports sent today. 148 149 This clears the contents of the rate limiting directory which has 150 the effect of reseting our count of crash reports sent. 151 """ 152 utils.system('rm -rf ' + self._CRASH_SENDER_RATE_DIR) 153 154 155 def _clear_spooled_crashes(self): 156 """Clears system and user crash directories. 157 158 This will remove all crash reports which are waiting to be sent. 159 """ 160 utils.system('rm -rf ' + self._SYSTEM_CRASH_DIR) 161 utils.system('rm -rf %s %s' % (self._USER_CRASH_DIRS, 162 self._FALLBACK_USER_CRASH_DIR)) 163 164 165 def _kill_running_sender(self): 166 """Kill the the crash_sender process if running. 167 168 We use the PID file to find the process ID, then kill it with signal 9. 169 """ 170 if not os.path.exists(self._CRASH_SENDER_RUN_PATH): 171 return 172 running_pid = int(utils.read_file(self._CRASH_SENDER_RUN_PATH)) 173 logging.warning('Detected running crash sender (%d), killing', 174 running_pid) 175 utils.system('kill -9 %d' % running_pid) 176 os.remove(self._CRASH_SENDER_RUN_PATH) 177 178 179 def _set_sending_mock(self, mock_enabled, send_success=True): 180 """Enables / disables mocking of the sending process. 181 182 This uses the _MOCK_CRASH_SENDING file to achieve its aims. See notes 183 at the top. 184 185 @param mock_enabled: If True, mocking is enabled, else it is disabled. 186 @param send_success: If mock_enabled this is True for the mocking to 187 indicate success, False to indicate failure. 188 """ 189 if mock_enabled: 190 if send_success: 191 data = '' 192 else: 193 data = '1' 194 logging.info('Setting sending mock') 195 utils.open_write_close(self._MOCK_CRASH_SENDING, data) 196 else: 197 utils.system('rm -f ' + self._MOCK_CRASH_SENDING) 198 199 200 def _set_consent(self, has_consent): 201 """Sets whether or not we have consent to send crash reports. 202 203 This creates or deletes the _CONSENT_FILE to control whether 204 crash_sender will consider that it has consent to send crash reports. 205 It also copies a policy blob with the proper policy setting. 206 207 @param has_consent: True to indicate consent, False otherwise 208 """ 209 autotest_cros_dir = os.path.dirname(__file__) 210 if has_consent: 211 if os.path.isdir(constants.WHITELIST_DIR): 212 # Create policy file that enables metrics/consent. 213 shutil.copy('%s/mock_metrics_on.policy' % autotest_cros_dir, 214 constants.SIGNED_POLICY_FILE) 215 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 216 constants.OWNER_KEY_FILE) 217 # Create deprecated consent file. This is created *after* the 218 # policy file in order to avoid a race condition where chrome 219 # might remove the consent file if the policy's not set yet. 220 # We create it as a temp file first in order to make the creation 221 # of the consent file, owned by chronos, atomic. 222 # See crosbug.com/18413. 223 temp_file = self._CONSENT_FILE + '.tmp'; 224 utils.open_write_close(temp_file, 'test-consent') 225 utils.system('chown chronos:chronos "%s"' % (temp_file)) 226 shutil.move(temp_file, self._CONSENT_FILE) 227 logging.info('Created ' + self._CONSENT_FILE) 228 else: 229 if os.path.isdir(constants.WHITELIST_DIR): 230 # Create policy file that disables metrics/consent. 231 shutil.copy('%s/mock_metrics_off.policy' % autotest_cros_dir, 232 constants.SIGNED_POLICY_FILE) 233 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 234 constants.OWNER_KEY_FILE) 235 # Remove deprecated consent file. 236 utils.system('rm -f "%s"' % (self._CONSENT_FILE)) 237 238 239 def _set_crash_test_in_progress(self, in_progress): 240 if in_progress: 241 utils.open_write_close(self._CRASH_TEST_IN_PROGRESS, 'in-progress') 242 logging.info('Created ' + self._CRASH_TEST_IN_PROGRESS) 243 else: 244 utils.system('rm -f "%s"' % (self._CRASH_TEST_IN_PROGRESS)) 245 246 247 def _get_pushed_consent_file_path(self): 248 """Returns filename of the pushed consent file.""" 249 return os.path.join(self.bindir, 'pushed_consent') 250 251 252 def _get_pushed_policy_file_path(self): 253 """Returns filename of the pushed policy file.""" 254 return os.path.join(self.bindir, 'pushed_policy') 255 256 257 def _get_pushed_owner_key_file_path(self): 258 """Returns filename of the pushed owner.key file.""" 259 return os.path.join(self.bindir, 'pushed_owner_key') 260 261 262 def _push_consent(self): 263 """Push the consent file, thus disabling consent. 264 265 The consent files can be created in the new test if required. Call 266 _pop_consent() to restore the original state. 267 """ 268 if os.path.exists(self._CONSENT_FILE): 269 shutil.move(self._CONSENT_FILE, 270 self._get_pushed_consent_file_path()) 271 if os.path.exists(constants.SIGNED_POLICY_FILE): 272 shutil.move(constants.SIGNED_POLICY_FILE, 273 self._get_pushed_policy_file_path()) 274 if os.path.exists(constants.OWNER_KEY_FILE): 275 shutil.move(constants.OWNER_KEY_FILE, 276 self._get_pushed_owner_key_file_path()) 277 278 279 def _pop_consent(self): 280 """Pop the consent files, enabling/disabling consent as it was before 281 we pushed the consent.""" 282 if os.path.exists(self._get_pushed_consent_file_path()): 283 shutil.move(self._get_pushed_consent_file_path(), 284 self._CONSENT_FILE) 285 else: 286 utils.system('rm -f "%s"' % self._CONSENT_FILE) 287 if os.path.exists(self._get_pushed_policy_file_path()): 288 shutil.move(self._get_pushed_policy_file_path(), 289 constants.SIGNED_POLICY_FILE) 290 else: 291 utils.system('rm -f "%s"' % constants.SIGNED_POLICY_FILE) 292 if os.path.exists(self._get_pushed_owner_key_file_path()): 293 shutil.move(self._get_pushed_owner_key_file_path(), 294 constants.OWNER_KEY_FILE) 295 else: 296 utils.system('rm -f "%s"' % constants.OWNER_KEY_FILE) 297 298 299 def _get_crash_dir(self, username): 300 """Returns full path to the crash directory for a given username 301 302 This only really works (currently) when no one is logged in. That 303 is OK (currently) as the only test that uses this runs when no one 304 is actually logged in. 305 306 @param username: username to use: 307 'chronos': Returns user crash directory. 308 'root': Returns system crash directory. 309 """ 310 if username == 'chronos': 311 return self._FALLBACK_USER_CRASH_DIR 312 else: 313 return self._SYSTEM_CRASH_DIR 314 315 316 def _initialize_crash_reporter(self): 317 """Start up the crash reporter.""" 318 utils.system('%s --init --nounclean_check' % self._CRASH_REPORTER_PATH) 319 # Completely disable crash_reporter from generating crash dumps 320 # while any tests are running, otherwise a crashy system can make 321 # these tests flaky. 322 self.enable_crash_filtering('none') 323 324 325 def get_crash_dir_name(self, name): 326 """Return the full path for |name| inside the system crash directory.""" 327 return os.path.join(self._SYSTEM_CRASH_DIR, name) 328 329 330 def write_crash_dir_entry(self, name, contents): 331 """Writes an empty file to the system crash directory. 332 333 This writes a file to _SYSTEM_CRASH_DIR with the given name. This is 334 used to insert new crash dump files for testing purposes. 335 336 @param name: Name of file to write. 337 @param contents: String to write to the file. 338 """ 339 entry = self.get_crash_dir_name(name) 340 if not os.path.exists(self._SYSTEM_CRASH_DIR): 341 os.makedirs(self._SYSTEM_CRASH_DIR) 342 utils.open_write_close(entry, contents) 343 return entry 344 345 346 def write_fake_meta(self, name, exec_name, payload, log=None, 347 complete=True): 348 """Writes a fake meta entry to the system crash directory. 349 350 @param name: Name of file to write. 351 @param exec_name: Value for exec_name item. 352 @param payload: Value for payload item. 353 @param log: Value for log item. 354 @param complete: True to close off the record, otherwise leave it 355 incomplete. 356 """ 357 last_line = '' 358 if complete: 359 last_line = 'done=1\n' 360 contents = ('exec_name=%s\n' 361 'ver=my_ver\n' 362 'payload=%s\n' 363 '%s' % (exec_name, payload, 364 last_line)) 365 if log: 366 contents = ('log=%s\n' % log) + contents 367 return self.write_crash_dir_entry(name, contents) 368 369 370 def _prepare_sender_one_crash(self, 371 send_success, 372 reports_enabled, 373 report): 374 """Create metadata for a fake crash report. 375 376 This enabled mocking of the crash sender, then creates a fake 377 crash report for testing purposes. 378 379 @param send_success: True to make the crash_sender success, False to 380 make it fail. 381 @param reports_enabled: True to enable consent to that reports will be 382 sent. 383 @param report: Report to use for crash, if None we create one. 384 """ 385 self._set_sending_mock(mock_enabled=True, send_success=send_success) 386 self._set_consent(reports_enabled) 387 if report is None: 388 # Use the same file format as crash does normally: 389 # <basename>.#.#.#.meta 390 payload = self.write_crash_dir_entry( 391 '%s.dmp' % self._FAKE_TEST_BASENAME, '') 392 report = self.write_fake_meta( 393 '%s.meta' % self._FAKE_TEST_BASENAME, 'fake', payload) 394 return report 395 396 397 def _parse_sender_output(self, output): 398 """Parse the log output from the crash_sender script. 399 400 This script can run on the logs from either a mocked or true 401 crash send. 402 403 @param output: output from the script 404 405 @returns A dictionary with these values: 406 error_type: an error type, if given 407 exec_name: name of executable which crashed 408 image_type: type of image ("dev","force-official",...), if given 409 boot_mode: current boot mode ("dev",...), if given 410 meta_path: path to the report metadata file 411 output: the output from the script, copied 412 report_kind: kind of report sent (minidump vs kernel) 413 send_attempt: did the script attempt to send a crash. 414 send_success: if it attempted, was the crash send successful. 415 sig: signature of the report, if given. 416 sleep_time: if it attempted, how long did it sleep before 417 sending (if mocked, how long would it have slept) 418 """ 419 sleep_match = re.search('Scheduled to send in (\d+)s', output) 420 send_attempt = sleep_match is not None 421 if send_attempt: 422 sleep_time = int(sleep_match.group(1)) 423 else: 424 sleep_time = None 425 426 meta_match = re.search('Metadata: (\S+) \((\S+)\)', output) 427 if meta_match: 428 meta_path = meta_match.group(1) 429 report_kind = meta_match.group(2) 430 else: 431 meta_path = None 432 report_kind = None 433 434 payload_match = re.search('Payload: (\S+)', output) 435 if payload_match: 436 report_payload = payload_match.group(1) 437 else: 438 report_payload = None 439 440 exec_name_match = re.search('Exec name: (\S+)', output) 441 if exec_name_match: 442 exec_name = exec_name_match.group(1) 443 else: 444 exec_name = None 445 446 sig_match = re.search('sig: (\S+)', output) 447 if sig_match: 448 sig = sig_match.group(1) 449 else: 450 sig = None 451 452 error_type_match = re.search('Error type: (\S+)', output) 453 if error_type_match: 454 error_type = error_type_match.group(1) 455 else: 456 error_type = None 457 458 image_type_match = re.search('Image type: (\S+)', output) 459 if image_type_match: 460 image_type = image_type_match.group(1) 461 else: 462 image_type = None 463 464 boot_mode_match = re.search('Boot mode: (\S+)', output) 465 if boot_mode_match: 466 boot_mode = boot_mode_match.group(1) 467 else: 468 boot_mode = None 469 470 send_success = 'Mocking successful send' in output 471 return {'exec_name': exec_name, 472 'report_kind': report_kind, 473 'meta_path': meta_path, 474 'report_payload': report_payload, 475 'send_attempt': send_attempt, 476 'send_success': send_success, 477 'sig': sig, 478 'error_type': error_type, 479 'image_type': image_type, 480 'boot_mode': boot_mode, 481 'sleep_time': sleep_time, 482 'output': output} 483 484 485 def wait_for_sender_completion(self): 486 """Wait for crash_sender to complete. 487 488 Wait for no crash_sender's last message to be placed in the 489 system log before continuing and for the process to finish. 490 Otherwise we might get only part of the output.""" 491 utils.poll_for_condition( 492 lambda: self._log_reader.can_find('crash_sender done.'), 493 timeout=60, 494 exception=error.TestError( 495 'Timeout waiting for crash_sender to emit done: ' + 496 self._log_reader.get_logs())) 497 utils.poll_for_condition( 498 lambda: utils.system('pgrep crash_sender', 499 ignore_status=True) != 0, 500 timeout=60, 501 exception=error.TestError( 502 'Timeout waiting for crash_sender to finish: ' + 503 self._log_reader.get_logs())) 504 505 506 def _call_sender_one_crash(self, 507 send_success=True, 508 reports_enabled=True, 509 username='root', 510 report=None, 511 should_fail=False): 512 """Call the crash sender script to mock upload one crash. 513 514 @param send_success: Mock a successful send if true 515 @param reports_enabled: Has the user consented to sending crash reports. 516 @param username: user to emulate a crash from 517 @param report: report to use for crash, if None we create one. 518 519 @returns a dictionary describing the result with the keys 520 from _parse_sender_output, as well as: 521 report_exists: does the minidump still exist after calling 522 send script 523 rate_count: how many crashes have been uploaded in the past 524 24 hours. 525 """ 526 report = self._prepare_sender_one_crash(send_success, 527 reports_enabled, 528 report) 529 self._log_reader.set_start_by_current() 530 script_output = "" 531 try: 532 script_output = utils.system_output( 533 '/bin/sh -c "%s" 2>&1' % self._CRASH_SENDER_PATH, 534 ignore_status=should_fail) 535 except error.CmdError as err: 536 raise error.TestFail('"%s" returned an unexpected non-zero ' 537 'value (%s).' 538 % (err.command, err.result_obj.exit_status)) 539 540 self.wait_for_sender_completion() 541 output = self._log_reader.get_logs() 542 logging.debug('Crash sender message output:\n' + output) 543 544 if script_output != '': 545 logging.debug('crash_sender stdout/stderr: ' + script_output) 546 547 if os.path.exists(report): 548 report_exists = True 549 os.remove(report) 550 else: 551 report_exists = False 552 if os.path.exists(self._CRASH_SENDER_RATE_DIR): 553 rate_count = len(os.listdir(self._CRASH_SENDER_RATE_DIR)) 554 else: 555 rate_count = 0 556 557 result = self._parse_sender_output(output) 558 result['report_exists'] = report_exists 559 result['rate_count'] = rate_count 560 561 # Show the result for debugging but remove 'output' key 562 # since it's large and earlier in debug output. 563 debug_result = dict(result) 564 del debug_result['output'] 565 logging.debug('Result of send (besides output): %s', debug_result) 566 567 return result 568 569 570 def _replace_crash_reporter_filter_in(self, new_parameter): 571 """Replaces the --filter_in= parameter of the crash reporter. 572 573 The kernel is set up to call the crash reporter with the core dump 574 as stdin when a process dies. This function adds a filter to the 575 command line used to call the crash reporter. This is used to ignore 576 crashes in which we have no interest. 577 578 This removes any --filter_in= parameter and optionally replaces it 579 with a new one. 580 581 @param new_parameter: This is parameter to add to the command line 582 instead of the --filter_in=... that was there. 583 """ 584 core_pattern = utils.read_file(self._CORE_PATTERN)[:-1] 585 core_pattern = re.sub('--filter_in=\S*\s*', '', 586 core_pattern).rstrip() 587 if new_parameter: 588 core_pattern += ' ' + new_parameter 589 utils.system('echo "%s" > %s' % (core_pattern, self._CORE_PATTERN)) 590 591 592 def enable_crash_filtering(self, name): 593 """Add a --filter_in argument to the kernel core dump cmdline. 594 595 @param name: Filter text to use. This is passed as a --filter_in 596 argument to the crash reporter. 597 """ 598 self._replace_crash_reporter_filter_in('--filter_in=' + name) 599 600 601 def disable_crash_filtering(self): 602 """Remove the --filter_in argument from the kernel core dump cmdline. 603 604 Next time the crash reporter is invoked (due to a crash) it will not 605 receive a --filter_in paramter.""" 606 self._replace_crash_reporter_filter_in('') 607 608 609 @contextlib.contextmanager 610 def hold_crash_lock(self): 611 """A context manager to hold the crash sender lock.""" 612 with open(self._CRASH_SENDER_LOCK_PATH, 'w+') as f: 613 fcntl.flock(f.fileno(), fcntl.LOCK_EX) 614 try: 615 yield 616 finally: 617 fcntl.flock(f.fileno(), fcntl.LOCK_UN) 618 619 620 def initialize(self): 621 """Initalize the test.""" 622 test.test.initialize(self) 623 self._log_reader = cros_logging.make_system_log_reader() 624 self._leave_crash_sending = True 625 self._automatic_consent_saving = True 626 self.enable_crash_filtering('none') 627 self._set_crash_test_in_progress(True) 628 629 630 def cleanup(self): 631 """Cleanup after the test. 632 633 We reset things back to the way we think they should be. This is 634 intended to allow the system to continue normal operation. 635 636 Some variables silently change the behavior: 637 _automatic_consent_saving: if True, we pop the consent file. 638 _leave_crash_sending: True to enable crash sending, False to 639 disable it 640 """ 641 self._reset_rate_limiting() 642 self._clear_spooled_crashes() 643 self._set_system_sending(self._leave_crash_sending) 644 self._set_sending_mock(mock_enabled=False) 645 if self._automatic_consent_saving: 646 self._pop_consent() 647 self.disable_crash_filtering() 648 self._set_crash_test_in_progress(False) 649 test.test.cleanup(self) 650 651 652 def run_crash_tests(self, 653 test_names, 654 initialize_crash_reporter=False, 655 clear_spool_first=True, 656 must_run_all=True): 657 """Run crash tests defined in this class. 658 659 @param test_names: Array of test names. 660 @param initialize_crash_reporter: Should set up crash reporter for every 661 run. 662 @param clear_spool_first: Clear all spooled user/system crashes before 663 starting the test. 664 @param must_run_all: Should make sure every test in this class is 665 mentioned in test_names. 666 """ 667 if self._automatic_consent_saving: 668 self._push_consent() 669 670 if must_run_all: 671 # Sanity check test_names is complete 672 for attr in dir(self): 673 if attr.find('_test_') == 0: 674 test_name = attr[6:] 675 if not test_name in test_names: 676 raise error.TestError('Test %s is missing' % test_name) 677 678 for test_name in test_names: 679 logging.info(('=' * 20) + ('Running %s' % test_name) + ('=' * 20)) 680 if initialize_crash_reporter: 681 self._initialize_crash_reporter() 682 # Disable crash_sender from running, kill off any running ones, but 683 # set environment so crash_sender may run as a child process. 684 self._set_system_sending(False) 685 self._set_child_sending(True) 686 self._kill_running_sender() 687 self._reset_rate_limiting() 688 # Default to not overriding for unofficial versions. 689 self._set_force_official(False) 690 # Default to not pretending we're in developer mode. 691 self._set_mock_developer_mode(False) 692 if clear_spool_first: 693 self._clear_spooled_crashes() 694 695 # Call the test function 696 getattr(self, '_test_' + test_name)() 697