• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import contextlib
6import fcntl
7import glob
8import logging
9import os
10import random
11import re
12import shutil
13import time
14
15import common
16from autotest_lib.client.bin import test, utils
17from autotest_lib.client.common_lib import error
18from autotest_lib.client.cros import constants, cros_logging
19
20
21class CrashTest(test.test):
22    """
23    This class deals with running crash tests, which are tests which crash a
24    user-space program (or the whole machine) and generate a core dump. We
25    want to check that the correct crash dump is available and can be
26    retrieved.
27
28    Chromium OS has a crash sender which checks for new crash data and sends
29    it to a server. This crash data is used to track software quality and find
30    bugs. The system crash sender normally is always running, but can be paused
31    by creating _PAUSE_FILE. When crash sender sees this, it pauses operation.
32
33    For testing purposes we sometimes want to run the crash sender manually.
34    In this case we can pass the --ignore_pause_file flag and run the crash
35    sender manually.
36
37    Also for testing we sometimes want to mock out the crash sender, and just
38    have it pretend to succeed or fail. The _MOCK_CRASH_SENDING file is used
39    for this. If it doesn't exist, then the crash sender runs normally. If
40    it exists but is empty, the crash sender will succeed (but actually do
41    nothing). If the file contains something, then the crash sender will fail.
42
43    If the user consents to sending crash tests, then the _CONSENT_FILE will
44    exist in the home directory. This test needs to create this file for the
45    crash sending to work. The metrics daemon caches the consent state for
46    1 second, so we need to sleep for more than that after changing it to be
47    sure it picks up the change.
48
49    Crash reports are rate limited to a certain number of reports each 24
50    hours. If the maximum number has already been sent then reports are held
51    until later. This is administered by a directory _CRASH_SENDER_RATE_DIR
52    which contains one temporary file for each time a report is sent.
53
54    The class provides the ability to push a consent file. This disables
55    consent for this test but allows it to be popped back at later. This
56    makes nested tests easier. If _automatic_consent_saving is True (the
57    default) then consent will be pushed at the start and popped at the end.
58
59    Interesting variables:
60        _log_reader: the log reader used for reading log files
61        _leave_crash_sending: True to enable crash sending on exit from the
62            test, False to disable it. (Default True).
63        _automatic_consent_saving: True to push the consent at the start of
64            the test and pop it afterwards. (Default True).
65
66    Useful places to look for more information are:
67
68    chromeos/src/platform/crash-reporter/crash_sender
69        - sender script which crash crash reporter to create reports, then
70
71    chromeos/src/platform/crash-reporter/
72        - crash reporter program
73    """
74
75
76    _CONSENT_FILE = '/home/chronos/Consent To Send Stats'
77    _CORE_PATTERN = '/proc/sys/kernel/core_pattern'
78    _LOCK_CORE_PATTERN = '/proc/sys/kernel/lock_core_pattern'
79    _CRASH_REPORTER_PATH = '/sbin/crash_reporter'
80    _CRASH_SENDER_PATH = '/sbin/crash_sender'
81    _CRASH_SENDER_RATE_DIR = '/var/lib/crash_sender'
82    _CRASH_SENDER_LOCK_PATH = '/run/lock/crash_sender'
83    _CRASH_RUN_STATE_DIR = '/run/crash_reporter'
84    _CRASH_TEST_IN_PROGRESS = _CRASH_RUN_STATE_DIR + '/crash-test-in-progress'
85    _MOCK_CRASH_SENDING = _CRASH_RUN_STATE_DIR + '/mock-crash-sending'
86    _PAUSE_FILE = '/var/lib/crash_sender_paused'
87    _SYSTEM_CRASH_DIR = '/var/spool/crash'
88    _FALLBACK_USER_CRASH_DIR = '/home/chronos/crash'
89    _EARLY_BOOT_CRASH_DIR = '/mnt/stateful_partition/unencrypted/preserve/crash'
90    _USER_CRASH_DIRS = '/home/chronos/u-*/crash'
91    _USER_CRASH_DIR_REGEX = re.compile('/home/chronos/u-([a-f0-9]+)/crash')
92
93    # Matches kDefaultMaxUploadBytes
94    _MAX_CRASH_SIZE = 1024 * 1024
95
96    # Use the same file format as crash does normally:
97    # <basename>.#.#.#.meta
98    _FAKE_TEST_BASENAME = 'fake.1.2.3'
99
100    def _set_system_sending(self, is_enabled):
101        """Sets whether or not the system crash_sender is allowed to run.
102
103        This is done by creating or removing _PAUSE_FILE.
104
105        crash_sender may still be allowed to run if _call_sender_one_crash is
106        called with 'ignore_pause=True'.
107
108        @param is_enabled: True to enable crash_sender, False to disable it.
109        """
110        if is_enabled:
111            if os.path.exists(self._PAUSE_FILE):
112                os.remove(self._PAUSE_FILE)
113        else:
114            utils.system('touch ' + self._PAUSE_FILE)
115
116
117    def _reset_rate_limiting(self):
118        """Reset the count of crash reports sent today.
119
120        This clears the contents of the rate limiting directory which has
121        the effect of reseting our count of crash reports sent.
122        """
123        utils.system('rm -rf ' + self._CRASH_SENDER_RATE_DIR)
124
125
126    def _clear_spooled_crashes(self):
127        """Clears system and user crash directories.
128
129        This will remove all crash reports which are waiting to be sent.
130        """
131        utils.system('rm -rf ' + self._SYSTEM_CRASH_DIR)
132        utils.system('rm -rf ' + self._EARLY_BOOT_CRASH_DIR)
133        utils.system('rm -rf %s %s' % (self._USER_CRASH_DIRS,
134                                       self._FALLBACK_USER_CRASH_DIR))
135
136
137    def _kill_running_sender(self):
138        """Kill the the crash_sender process if running."""
139        utils.system('pkill -9 -e --exact crash_sender', ignore_status=True)
140
141
142    def _set_sending_mock(self, mock_enabled, send_success=True):
143        """Enables / disables mocking of the sending process.
144
145        This uses the _MOCK_CRASH_SENDING file to achieve its aims. See notes
146        at the top.
147
148        @param mock_enabled: If True, mocking is enabled, else it is disabled.
149        @param send_success: If mock_enabled this is True for the mocking to
150                indicate success, False to indicate failure.
151        """
152        if mock_enabled:
153            if send_success:
154                data = ''
155            else:
156                data = '1'
157            logging.info('Setting sending mock')
158            utils.open_write_close(self._MOCK_CRASH_SENDING, data)
159        else:
160            utils.system('rm -f ' + self._MOCK_CRASH_SENDING)
161
162
163    def _set_consent(self, has_consent):
164        """Sets whether or not we have consent to send crash reports.
165
166        This creates or deletes the _CONSENT_FILE to control whether
167        crash_sender will consider that it has consent to send crash reports.
168        It also copies a policy blob with the proper policy setting.
169
170        @param has_consent: True to indicate consent, False otherwise
171        """
172        autotest_cros_dir = os.path.join(os.path.dirname(__file__), '..')
173        if has_consent:
174            if os.path.isdir(constants.WHITELIST_DIR):
175                # Create policy file that enables metrics/consent.
176                shutil.copy('%s/mock_metrics_on.policy' % autotest_cros_dir,
177                            constants.SIGNED_POLICY_FILE)
178                shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir,
179                            constants.OWNER_KEY_FILE)
180            # Create deprecated consent file.  This is created *after* the
181            # policy file in order to avoid a race condition where chrome
182            # might remove the consent file if the policy's not set yet.
183            # We create it as a temp file first in order to make the creation
184            # of the consent file, owned by chronos, atomic.
185            # See crosbug.com/18413.
186            temp_file = self._CONSENT_FILE + '.tmp';
187            utils.open_write_close(temp_file, 'test-consent')
188            utils.system('chown chronos:chronos "%s"' % (temp_file))
189            shutil.move(temp_file, self._CONSENT_FILE)
190            logging.info('Created %s', self._CONSENT_FILE)
191        else:
192            if os.path.isdir(constants.WHITELIST_DIR):
193                # Create policy file that disables metrics/consent.
194                shutil.copy('%s/mock_metrics_off.policy' % autotest_cros_dir,
195                            constants.SIGNED_POLICY_FILE)
196                shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir,
197                            constants.OWNER_KEY_FILE)
198            # Remove deprecated consent file.
199            utils.system('rm -f "%s"' % (self._CONSENT_FILE))
200        # Ensure cached consent state is updated.
201        time.sleep(2)
202
203
204    def _set_crash_test_in_progress(self, in_progress):
205        if in_progress:
206            utils.open_write_close(self._CRASH_TEST_IN_PROGRESS, 'in-progress')
207            logging.info('Created %s', self._CRASH_TEST_IN_PROGRESS)
208        else:
209            utils.system('rm -f "%s"' % (self._CRASH_TEST_IN_PROGRESS))
210
211
212    def _get_pushed_consent_file_path(self):
213        """Returns filename of the pushed consent file."""
214        return os.path.join(self.bindir, 'pushed_consent')
215
216
217    def _get_pushed_policy_file_path(self):
218        """Returns filename of the pushed policy file."""
219        return os.path.join(self.bindir, 'pushed_policy')
220
221
222    def _get_pushed_owner_key_file_path(self):
223        """Returns filename of the pushed owner.key file."""
224        return os.path.join(self.bindir, 'pushed_owner_key')
225
226
227    def _push_consent(self):
228        """Push the consent file, thus disabling consent.
229
230        The consent files can be created in the new test if required. Call
231        _pop_consent() to restore the original state.
232        """
233        if os.path.exists(self._CONSENT_FILE):
234            shutil.move(self._CONSENT_FILE,
235                        self._get_pushed_consent_file_path())
236        if os.path.exists(constants.SIGNED_POLICY_FILE):
237            shutil.move(constants.SIGNED_POLICY_FILE,
238                        self._get_pushed_policy_file_path())
239        if os.path.exists(constants.OWNER_KEY_FILE):
240            shutil.move(constants.OWNER_KEY_FILE,
241                        self._get_pushed_owner_key_file_path())
242        # Ensure cached consent state is updated.
243        time.sleep(2)
244
245
246    def _pop_consent(self):
247        """Pop the consent files, enabling/disabling consent as it was before
248        we pushed the consent."""
249        if os.path.exists(self._get_pushed_consent_file_path()):
250            shutil.move(self._get_pushed_consent_file_path(),
251                        self._CONSENT_FILE)
252        else:
253            utils.system('rm -f "%s"' % self._CONSENT_FILE)
254        if os.path.exists(self._get_pushed_policy_file_path()):
255            shutil.move(self._get_pushed_policy_file_path(),
256                        constants.SIGNED_POLICY_FILE)
257        else:
258            utils.system('rm -f "%s"' % constants.SIGNED_POLICY_FILE)
259        if os.path.exists(self._get_pushed_owner_key_file_path()):
260            shutil.move(self._get_pushed_owner_key_file_path(),
261                        constants.OWNER_KEY_FILE)
262        else:
263            utils.system('rm -f "%s"' % constants.OWNER_KEY_FILE)
264        # Ensure cached consent state is updated.
265        time.sleep(2)
266
267
268    def _get_crash_dir(self, username, force_user_crash_dir=False):
269        """Returns crash directory for process running as the given user.
270
271        @param username: Unix user of the crashing process.
272        @param force_user_crash_dir: Regardless of |username|, return the crash
273                                     directory of the current user session, or
274                                     the fallback directory if no sessions.
275        """
276        if username in ('root', 'crash') and not force_user_crash_dir:
277            return self._SYSTEM_CRASH_DIR
278        else:
279            dirs = glob.glob(self._USER_CRASH_DIRS)
280            return dirs[0] if dirs else self._FALLBACK_USER_CRASH_DIR
281
282
283    def _canonicalize_crash_dir(self, crash_dir):
284        """Converts /home/chronos crash directory to /home/user counterpart.
285
286        @param crash_dir: A path of the form /home/chronos/u-<hash>/crash.
287        @returns /home/user/<hash>/crash, or |crash_dir| on form mismatch.
288        """
289        match = re.match(self._USER_CRASH_DIR_REGEX, crash_dir)
290        return ('/home/user/%s/crash' % match.group(1)) if match else crash_dir
291
292
293    def _initialize_crash_reporter(self, lock_core_pattern):
294        """Start up the crash reporter.
295
296        @param lock_core_pattern: lock core pattern during initialization.
297        """
298
299        if not lock_core_pattern:
300            self._set_crash_test_in_progress(False)
301        utils.system('%s --init' % self._CRASH_REPORTER_PATH)
302        if not lock_core_pattern:
303            self._set_crash_test_in_progress(True)
304            # Completely disable crash_reporter from generating crash dumps
305            # while any tests are running, otherwise a crashy system can make
306            # these tests flaky.
307            self.enable_crash_filtering('none')
308
309
310    def get_crash_dir_name(self, name):
311        """Return the full path for |name| inside the system crash directory."""
312        return os.path.join(self._SYSTEM_CRASH_DIR, name)
313
314
315    def write_crash_dir_entry(self, name, contents):
316        """Writes a file to the system crash directory.
317
318        This writes a file to _SYSTEM_CRASH_DIR with the given name. This is
319        used to insert new crash dump files for testing purposes.
320
321        @param name: Name of file to write.
322        @param contents: String to write to the file.
323        """
324        entry = self.get_crash_dir_name(name)
325        if not os.path.exists(self._SYSTEM_CRASH_DIR):
326            os.makedirs(self._SYSTEM_CRASH_DIR)
327        utils.open_write_close(entry, contents)
328        return entry
329
330
331    def write_fake_meta(self, name, exec_name, payload, complete=True):
332        """Writes a fake meta entry to the system crash directory.
333
334        @param name: Name of file to write.
335        @param exec_name: Value for exec_name item.
336        @param payload: Value for payload item.
337        @param complete: True to close off the record, otherwise leave it
338                incomplete.
339        """
340        last_line = ''
341        if complete:
342            last_line = 'done=1\n'
343        contents = ('exec_name=%s\n'
344                    'ver=my_ver\n'
345                    'payload=%s\n'
346                    '%s' % (exec_name, payload,
347                            last_line))
348        return self.write_crash_dir_entry(name, contents)
349
350    def _get_dmp_contents(self):
351        """Creates the contents of the dmp file for our made crashes.
352
353        The dmp file contents are deliberately large and hard-to-compress. This
354        ensures logging_CrashSender hits its bytes/day cap before its sends/day
355        cap.
356        """
357        return bytearray(
358                [random.randint(0, 255) for n in range(self._MAX_CRASH_SIZE)])
359
360
361    def _prepare_sender_one_crash(self,
362                                  send_success,
363                                  reports_enabled,
364                                  report):
365        """Create metadata for a fake crash report.
366
367        This enabled mocking of the crash sender, then creates a fake
368        crash report for testing purposes.
369
370        @param send_success: True to make the crash_sender success, False to
371                make it fail.
372        @param reports_enabled: True to enable consent to that reports will be
373                sent.
374        @param report: Report to use for crash, if None we create one.
375        """
376        self._set_sending_mock(mock_enabled=True, send_success=send_success)
377        self._set_consent(reports_enabled)
378        if report is None:
379            # Use the same file format as crash does normally:
380            # <basename>.#.#.#.meta
381            payload = self.write_crash_dir_entry(
382                '%s.dmp' % self._FAKE_TEST_BASENAME, self._get_dmp_contents())
383            report = self.write_fake_meta(
384                '%s.meta' % self._FAKE_TEST_BASENAME, 'fake', payload)
385        return report
386
387
388    def _parse_sender_output(self, output):
389        """Parse the log output from the crash_sender script.
390
391        This script can run on the logs from either a mocked or true
392        crash send. It looks for one and only one crash from output.
393        Non-crash anomalies should be ignored since there're just noise
394        during running the test.
395
396        @param output: output from the script
397
398        @returns A dictionary with these values:
399            exec_name: name of executable which crashed
400            image_type: type of image ("dev","test",...), if given
401            boot_mode: current boot mode ("dev",...), if given
402            meta_path: path to the report metadata file
403            output: the output from the script, copied
404            report_kind: kind of report sent (minidump vs kernel)
405            send_attempt: did the script attempt to send a crash.
406            send_success: if it attempted, was the crash send successful.
407            sig: signature of the report, if given.
408            sleep_time: if it attempted, how long did it sleep before
409              sending (if mocked, how long would it have slept)
410        """
411        anomaly_types = (
412            'kernel_suspend_warning',
413            'kernel_warning',
414            'kernel_wifi_warning',
415            'selinux_violation',
416            'service_failure',
417        )
418
419        def crash_sender_search(regexp, output):
420            """Narrow search to lines from crash_sender."""
421            return re.search(r'crash_sender\[\d+\]:\s+' + regexp, output)
422
423        before_first_crash = None
424        while True:
425            crash_header = crash_sender_search(
426                'Considering metadata (\S+)',
427                output
428            )
429            if not crash_header:
430                break
431            if before_first_crash is None:
432                before_first_crash = output[:crash_header.start()]
433            meta_considered = crash_header.group(1)
434            is_anomaly = any(x in meta_considered for x in anomaly_types)
435            if is_anomaly:
436                # If it's an anomaly, skip this header, and look for next
437                # one.
438                output = output[crash_header.end():]
439            else:
440                # If it's not an anomaly, skip everything before this
441                # header.
442                output = output[crash_header.start():]
443                break
444        if before_first_crash:
445            output = before_first_crash + output
446        logging.debug('Filtered sender output to parse:\n%s', output)
447
448        sleep_match = crash_sender_search('Scheduled to send in (\d+)s', output)
449        send_attempt = sleep_match is not None
450        if send_attempt:
451            sleep_time = int(sleep_match.group(1))
452        else:
453            sleep_time = None
454
455        meta_match = crash_sender_search('Metadata: (\S+) \((\S+)\)', output)
456        if meta_match:
457            meta_path = meta_match.group(1)
458            report_kind = meta_match.group(2)
459        else:
460            meta_path = None
461            report_kind = None
462
463        payload_match = crash_sender_search('Payload: (\S+)', output)
464        if payload_match:
465            report_payload = payload_match.group(1)
466        else:
467            report_payload = None
468
469        exec_name_match = crash_sender_search('Exec name: (\S+)', output)
470        if exec_name_match:
471            exec_name = exec_name_match.group(1)
472        else:
473            exec_name = None
474
475        sig_match = crash_sender_search('sig: (\S+)', output)
476        if sig_match:
477            sig = sig_match.group(1)
478        else:
479            sig = None
480
481        image_type_match = crash_sender_search('Image type: (\S+)', output)
482        if image_type_match:
483            image_type = image_type_match.group(1)
484        else:
485            image_type = None
486
487        boot_mode_match = crash_sender_search('Boot mode: (\S+)', output)
488        if boot_mode_match:
489            boot_mode = boot_mode_match.group(1)
490        else:
491            boot_mode = None
492
493        send_success = 'Mocking successful send' in output
494        return {'exec_name': exec_name,
495                'report_kind': report_kind,
496                'meta_path': meta_path,
497                'report_payload': report_payload,
498                'send_attempt': send_attempt,
499                'send_success': send_success,
500                'sig': sig,
501                'image_type': image_type,
502                'boot_mode': boot_mode,
503                'sleep_time': sleep_time,
504                'output': output}
505
506
507    def wait_for_sender_completion(self):
508        """Wait for crash_sender to complete.
509
510        Wait for no crash_sender's last message to be placed in the
511        system log before continuing and for the process to finish.
512        Otherwise we might get only part of the output."""
513        utils.poll_for_condition(
514            lambda: self._log_reader.can_find('crash_sender done.'),
515            timeout=60,
516            exception=error.TestError(
517              'Timeout waiting for crash_sender to emit done: ' +
518              self._log_reader.get_logs()))
519        utils.poll_for_condition(
520            lambda: utils.system('pgrep crash_sender',
521                                 ignore_status=True) != 0,
522            timeout=60,
523            exception=error.TestError(
524                'Timeout waiting for crash_sender to finish: ' +
525                self._log_reader.get_logs()))
526
527
528    def _call_sender_one_crash(self,
529                               send_success=True,
530                               reports_enabled=True,
531                               report=None,
532                               should_fail=False,
533                               ignore_pause=True):
534        """Call the crash sender script to mock upload one crash.
535
536        @param send_success: Mock a successful send if true
537        @param reports_enabled: Has the user consented to sending crash reports.
538        @param report: report to use for crash, if None we create one.
539        @param should_fail: expect the crash_sender program to fail
540        @param ignore_pause: crash_sender should ignore pause file existence
541
542        @returns a dictionary describing the result with the keys
543          from _parse_sender_output, as well as:
544            report_exists: does the minidump still exist after calling
545              send script
546            rate_count: how many crashes have been uploaded in the past
547              24 hours.
548        """
549        report = self._prepare_sender_one_crash(send_success,
550                                                reports_enabled,
551                                                report)
552        self._log_reader.set_start_by_current()
553        script_output = ""
554        try:
555            script_output = utils.system_output(
556                '%s %s2>&1' % (self._CRASH_SENDER_PATH,
557                               "--ignore_pause_file " if ignore_pause else ""),
558                ignore_status=should_fail)
559        except error.CmdError as err:
560            raise error.TestFail('"%s" returned an unexpected non-zero '
561                                 'value (%s).'
562                                 % (err.command, err.result_obj.exit_status))
563
564        self.wait_for_sender_completion()
565        output = self._log_reader.get_logs()
566        logging.debug('Crash sender message output:\n %s', output)
567
568        if script_output != '':
569            logging.debug('crash_sender stdout/stderr: %s', script_output)
570
571        if os.path.exists(report):
572            report_exists = True
573            os.remove(report)
574        else:
575            report_exists = False
576        if os.path.exists(self._CRASH_SENDER_RATE_DIR):
577            rate_count = len([
578                name for name in os.listdir(self._CRASH_SENDER_RATE_DIR)
579                if os.path.isfile(os.path.join(self._CRASH_SENDER_RATE_DIR,
580                                               name))
581            ])
582        else:
583            rate_count = 0
584
585        result = self._parse_sender_output(output)
586        result['report_exists'] = report_exists
587        result['rate_count'] = rate_count
588
589        # Show the result for debugging but remove 'output' key
590        # since it's large and earlier in debug output.
591        debug_result = dict(result)
592        del debug_result['output']
593        logging.debug('Result of send (besides output): %s', debug_result)
594
595        return result
596
597
598    def _replace_crash_reporter_filter_in(self, new_parameter):
599        """Replaces the --filter_in= parameter of the crash reporter.
600
601        The kernel is set up to call the crash reporter with the core dump
602        as stdin when a process dies. This function adds a filter to the
603        command line used to call the crash reporter. This is used to ignore
604        crashes in which we have no interest.
605
606        This removes any --filter_in= parameter and optionally replaces it
607        with a new one.
608
609        @param new_parameter: This is parameter to add to the command line
610                instead of the --filter_in=... that was there.
611        """
612        core_pattern = utils.read_file(self._CORE_PATTERN)[:-1]
613        core_pattern = re.sub('--filter_in=\S*\s*', '',
614                              core_pattern).rstrip()
615        if new_parameter:
616            core_pattern += ' ' + new_parameter
617        utils.system('echo "%s" > %s' % (core_pattern, self._CORE_PATTERN))
618
619
620    def enable_crash_filtering(self, name):
621        """Add a --filter_in argument to the kernel core dump cmdline.
622
623        @param name: Filter text to use. This is passed as a --filter_in
624                argument to the crash reporter.
625        """
626        self._replace_crash_reporter_filter_in('--filter_in=' + name)
627
628
629    def disable_crash_filtering(self):
630        """Remove the --filter_in argument from the kernel core dump cmdline.
631
632        Next time the crash reporter is invoked (due to a crash) it will not
633        receive a --filter_in paramter."""
634        self._replace_crash_reporter_filter_in('')
635
636
637    @contextlib.contextmanager
638    def hold_crash_lock(self):
639        """A context manager to hold the crash sender lock."""
640        with open(self._CRASH_SENDER_LOCK_PATH, 'w+') as f:
641            fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
642            try:
643                yield
644            finally:
645                fcntl.lockf(f.fileno(), fcntl.LOCK_UN)
646
647
648    def initialize(self):
649        """Initalize the test."""
650        test.test.initialize(self)
651        self._log_reader = cros_logging.make_system_log_reader()
652        self._leave_crash_sending = True
653        self._automatic_consent_saving = True
654        self.enable_crash_filtering('none')
655        self._set_crash_test_in_progress(True)
656
657
658    def cleanup(self):
659        """Cleanup after the test.
660
661        We reset things back to the way we think they should be. This is
662        intended to allow the system to continue normal operation.
663
664        Some variables silently change the behavior:
665            _automatic_consent_saving: if True, we pop the consent file.
666            _leave_crash_sending: True to enable crash sending, False to
667                disable it
668        """
669        self._reset_rate_limiting()
670        self._clear_spooled_crashes()
671        self._set_system_sending(self._leave_crash_sending)
672        self._set_sending_mock(mock_enabled=False)
673        if self._automatic_consent_saving:
674            self._pop_consent()
675        self._set_crash_test_in_progress(False)
676        test.test.cleanup(self)
677
678
679    def run_crash_tests(self,
680                        test_names,
681                        initialize_crash_reporter=False,
682                        clear_spool_first=True,
683                        must_run_all=True,
684                        lock_core_pattern=False):
685        """Run crash tests defined in this class.
686
687        @param test_names: Array of test names.
688        @param initialize_crash_reporter: Should set up crash reporter for every
689                run.
690        @param clear_spool_first: Clear all spooled user/system crashes before
691                starting the test.
692        @param must_run_all: Should make sure every test in this class is
693                mentioned in test_names.
694        @param lock_core_pattern: Lock core_pattern while initializing
695                crash_reporter.
696        """
697        if self._automatic_consent_saving:
698            self._push_consent()
699
700        if must_run_all:
701            # Sanity check test_names is complete
702            for attr in dir(self):
703                if attr.find('_test_') == 0:
704                    test_name = attr[6:]
705                    if not test_name in test_names:
706                        raise error.TestError('Test %s is missing' % test_name)
707
708        for test_name in test_names:
709            logging.info(('=' * 20) + ('Running %s' % test_name) + ('=' * 20))
710            if initialize_crash_reporter:
711                self._initialize_crash_reporter(lock_core_pattern)
712            # Disable crash_sender from running, kill off any running ones.
713            # We set a flag to crash_sender when invoking it manually to avoid
714            # our invocations being paused.
715            self._set_system_sending(False)
716            self._kill_running_sender()
717            self._reset_rate_limiting()
718            if clear_spool_first:
719                self._clear_spooled_crashes()
720
721            # Call the test function
722            getattr(self, '_test_' + test_name)()
723
724        # Clear the intentional crashes, so that the server won't automatically
725        # report crash as failure.
726        self._clear_spooled_crashes()
727