1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import grp 6import logging 7import os 8import pwd 9import re 10import shutil 11import signal 12import stat 13import subprocess 14 15import crash_test 16from autotest_lib.client.bin import utils 17from autotest_lib.client.common_lib import error 18 19 20CRASHER = 'crasher_nobreakpad' 21 22 23class UserCrashTest(crash_test.CrashTest): 24 """ 25 Base class for tests that verify crash reporting for user processes. Shared 26 functionality includes installing a crasher executable, generating Breakpad 27 symbols, running the crasher process, and verifying collection and sending. 28 """ 29 30 31 # Every crash report needs one of these to be valid. 32 REPORT_REQUIRED_FILETYPES = {'meta'} 33 # Reports might have these and that's OK! 34 REPORT_OPTIONAL_FILETYPES = {'dmp', 'log', 'proclog', 'pslog'} 35 36 37 def setup(self): 38 """Copy the crasher source code under |srcdir| and build it.""" 39 src = os.path.join(os.path.dirname(__file__), 'crasher') 40 dest = os.path.join(self.srcdir, 'crasher') 41 shutil.copytree(src, dest) 42 43 os.chdir(dest) 44 utils.make() 45 46 47 def initialize(self, expected_tag='user', expected_version=None, 48 force_user_crash_dir=False): 49 """Initialize and configure the test. 50 51 @param expected_tag: Expected tag in crash_reporter log message. 52 @param expected_version: Expected version included in the crash report, 53 or None to use the Chrome OS version. 54 @param force_user_crash_dir: Always look for crash reports in the crash 55 directory of the current user session, or 56 the fallback directory if no sessions. 57 """ 58 crash_test.CrashTest.initialize(self) 59 self._expected_tag = expected_tag 60 self._expected_version = expected_version 61 self._force_user_crash_dir = force_user_crash_dir 62 63 64 def _prepare_crasher(self, root_path='/'): 65 """Extract the crasher and set its permissions. 66 67 crasher is only gzipped to subvert Portage stripping. 68 69 @param root_path: Root directory of the chroot environment in which the 70 crasher is installed and run. 71 """ 72 self._root_path = root_path 73 self._crasher_path = os.path.join(self.srcdir, 'crasher', CRASHER) 74 utils.system('cd %s; tar xzf crasher.tgz-unmasked' % 75 os.path.dirname(self._crasher_path)) 76 # Make sure all users (specifically chronos) have access to 77 # this directory and its decendents in order to run crasher 78 # executable as different users. 79 utils.system('chmod -R a+rx ' + self.bindir) 80 81 82 def _populate_symbols(self): 83 """Set up Breakpad's symbol structure. 84 85 Breakpad's minidump processor expects symbols to be in a directory 86 hierarchy: 87 <symbol-root>/<module_name>/<file_id>/<module_name>.sym 88 """ 89 self._symbol_dir = os.path.join(os.path.dirname(self._crasher_path), 90 'symbols') 91 utils.system('rm -rf %s' % self._symbol_dir) 92 os.mkdir(self._symbol_dir) 93 94 basename = os.path.basename(self._crasher_path) 95 utils.system('/usr/bin/dump_syms %s > %s.sym' % 96 (self._crasher_path, 97 basename)) 98 sym_name = '%s.sym' % basename 99 symbols = utils.read_file(sym_name) 100 # First line should be like: 101 # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX 102 # or 103 # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX 104 first_line = symbols.split('\n')[0] 105 tokens = first_line.split() 106 if tokens[0] != 'MODULE' or tokens[1] != 'Linux': 107 raise error.TestError('Unexpected symbols format: %s', first_line) 108 file_id = tokens[3] 109 target_dir = os.path.join(self._symbol_dir, basename, file_id) 110 os.makedirs(target_dir) 111 os.rename(sym_name, os.path.join(target_dir, sym_name)) 112 113 114 def _is_frame_in_stack(self, frame_index, module_name, 115 function_name, file_name, 116 line_number, stack): 117 """Search for frame entries in the given stack dump text. 118 119 A frame entry looks like (alone on a line): 120 16 crasher_nobreakpad!main [crasher.cc : 21 + 0xb] 121 122 Args: 123 frame_index: number of the stack frame (0 is innermost frame) 124 module_name: name of the module (executable or dso) 125 function_name: name of the function in the stack 126 file_name: name of the file containing the function 127 line_number: line number 128 stack: text string of stack frame entries on separate lines. 129 130 Returns: 131 Boolean indicating if an exact match is present. 132 133 Note: 134 We do not care about the full function signature - ie, is it 135 foo or foo(ClassA *). These are present in function names 136 pulled by dump_syms for Stabs but not for DWARF. 137 """ 138 regexp = (r'\n\s*%d\s+%s!%s.*\[\s*%s\s*:\s*%d\s.*\]' % 139 (frame_index, module_name, 140 function_name, file_name, 141 line_number)) 142 logging.info('Searching for regexp %s', regexp) 143 return re.search(regexp, stack) is not None 144 145 146 def _verify_stack(self, stack, basename, from_crash_reporter): 147 # Should identify cause as SIGSEGV at address 0x16. 148 logging.debug('minidump_stackwalk output:\n%s', stack) 149 150 # Look for a line like: 151 # Crash reason: SIGSEGV 152 # Crash reason: SIGSEGV /0x00000000 153 match = re.search(r'Crash reason:\s+([^\s]*)', stack) 154 expected_address = '0x16' 155 if not match or match.group(1) != 'SIGSEGV': 156 raise error.TestFail('Did not identify SIGSEGV cause') 157 match = re.search(r'Crash address:\s+(.*)', stack) 158 if not match or match.group(1) != expected_address: 159 raise error.TestFail('Did not identify crash address %s' % 160 expected_address) 161 162 # Should identify crash at *(char*)0x16 assignment line 163 if not self._is_frame_in_stack(0, basename, 164 'recbomb', 'bomb.cc', 9, stack): 165 raise error.TestFail('Did not show crash line on stack') 166 167 # Should identify recursion line which is on the stack 168 # for 15 levels 169 if not self._is_frame_in_stack(15, basename, 'recbomb', 170 'bomb.cc', 12, stack): 171 raise error.TestFail('Did not show recursion line on stack') 172 173 # Should identify main line 174 if not self._is_frame_in_stack(16, basename, 'main', 175 'crasher.cc', 24, stack): 176 raise error.TestFail('Did not show main on stack') 177 178 179 def _run_crasher_process(self, username, cause_crash=True, consent=True, 180 crasher_path=None, run_crasher=None, 181 expected_uid=None, expected_gid=None, 182 expected_exit_code=None, expected_reason=None): 183 """Runs the crasher process. 184 185 Will wait up to 10 seconds for crash_reporter to report the crash. 186 crash_reporter_caught will be marked as true when the "Received crash 187 notification message..." appears. While associated logs are likely to be 188 available at this point, the function does not guarantee this. 189 190 @param username: Unix user of the crasher process. 191 @param cause_crash: Whether the crasher should crash. 192 @param consent: Whether the user consents to crash reporting. 193 @param crasher_path: Path to which the crasher should be copied before 194 execution. Relative to |_root_path|. 195 @param run_crasher: A closure to override the default |crasher_command| 196 invocation. It should return a tuple describing the 197 process, where |pid| can be None if it should be 198 parsed from the |output|: 199 200 def run_crasher(username, crasher_command): 201 ... 202 return (exit_code, output, pid) 203 204 @param expected_uid: The uid the crash happens under. 205 @param expected_gid: The gid the crash happens under. 206 @param expected_exit_code: 207 @param expected_reason: 208 Expected information in crash_reporter log message. 209 210 @returns: 211 A dictionary with keys: 212 returncode: return code of the crasher 213 crashed: did the crasher return segv error code 214 crash_reporter_caught: did crash_reporter catch a segv 215 output: stderr output of the crasher process 216 """ 217 if crasher_path is None: 218 crasher_path = self._crasher_path 219 else: 220 dest = os.path.join(self._root_path, 221 crasher_path[os.path.isabs(crasher_path):]) 222 223 utils.system('cp -a "%s" "%s"' % (self._crasher_path, dest)) 224 225 # Limit to the first 15 characters of the crasher binary name because 226 # that's what the kernel invokes crash_reporter with. 227 self.enable_crash_filtering(os.path.basename(crasher_path)[:15]) 228 229 crasher_command = [] 230 231 if username == 'root': 232 if expected_exit_code is None: 233 expected_exit_code = -signal.SIGSEGV 234 else: 235 if expected_exit_code is None: 236 expected_exit_code = 128 + signal.SIGSEGV 237 238 if not run_crasher: 239 crasher_command.extend(['su', username, '-c']) 240 241 crasher_command.append(crasher_path) 242 basename = os.path.basename(crasher_path) 243 if not cause_crash: 244 crasher_command.append('--nocrash') 245 self._set_consent(consent) 246 247 logging.debug('Running crasher: %s', crasher_command) 248 249 if run_crasher: 250 (exit_code, output, pid) = run_crasher(username, crasher_command) 251 252 else: 253 crasher = subprocess.Popen(crasher_command, 254 stdout=subprocess.PIPE, 255 stderr=subprocess.PIPE) 256 257 output = crasher.communicate()[1] 258 exit_code = crasher.returncode 259 pid = None 260 261 logging.debug('Crasher output:\n%s', output) 262 263 if pid is None: 264 # Get the PID from the output, since |crasher.pid| may be su's PID. 265 match = re.search(r'pid=(\d+)', output) 266 if not match: 267 raise error.TestFail('Missing PID in crasher output') 268 pid = int(match.group(1)) 269 270 if expected_uid is None: 271 expected_uid = pwd.getpwnam(username).pw_uid 272 273 if expected_gid is None: 274 expected_gid = pwd.getpwnam(username).pw_gid 275 276 if expected_reason is None and consent: 277 expected_reason = 'handling' 278 279 if expected_reason is not None: 280 expected_message = (( 281 '[%s] Received crash notification for %s[%d] sig 11, user %d ' 282 'group %d (%s)') % 283 (self._expected_tag, basename, pid, 284 expected_uid, expected_gid, expected_reason)) 285 else: 286 # No consent; different message format. 287 expected_message = (( 288 'No consent. Not handling invocation: /sbin/crash_reporter ' 289 '--user=%d:11:%d:%d:%s') % 290 (pid, expected_uid, expected_gid, basename)) 291 292 # Wait until no crash_reporter is running. 293 utils.poll_for_condition( 294 lambda: utils.system('pgrep -f crash_reporter.*:%s' % basename, 295 ignore_status=True) != 0, 296 timeout=10, 297 exception=error.TestError( 298 'Timeout waiting for crash_reporter to finish: ' + 299 self._log_reader.get_logs())) 300 301 is_caught = False 302 try: 303 utils.poll_for_condition( 304 lambda: self._log_reader.can_find(expected_message), 305 timeout=5, 306 desc='Logs contain crash_reporter message: ' + expected_message) 307 is_caught = True 308 except utils.TimeoutError: 309 pass 310 311 result = {'crashed': exit_code == expected_exit_code, 312 'crash_reporter_caught': is_caught, 313 'output': output, 314 'returncode': exit_code} 315 logging.debug('Crasher process result: %s', result) 316 return result 317 318 319 def _check_crash_directory_permissions(self, crash_dir): 320 stat_info = os.stat(crash_dir) 321 user = pwd.getpwuid(stat_info.st_uid).pw_name 322 group = grp.getgrgid(stat_info.st_gid).gr_name 323 mode = stat.S_IMODE(stat_info.st_mode) 324 325 if crash_dir.startswith('/var/spool/crash'): 326 if stat.S_ISDIR(stat_info.st_mode): 327 utils.system('ls -l %s' % crash_dir) 328 for f in os.listdir(crash_dir): 329 self._check_crash_directory_permissions( 330 os.path.join(crash_dir, f)) 331 permitted_modes = set([0o2770]) 332 else: 333 permitted_modes = set([0o660, 0o640, 0o644]) 334 expected_user = 'root' 335 expected_group = 'crash-access' 336 else: 337 permitted_modes = set([0o2770]) 338 expected_user = 'chronos' 339 expected_group = 'crash-user-access' 340 341 if user != expected_user or group != expected_group: 342 raise error.TestFail( 343 'Expected %s.%s ownership of %s (actual %s.%s)' % 344 (expected_user, expected_group, crash_dir, user, group)) 345 if mode not in permitted_modes: 346 raise error.TestFail( 347 'Expected %s to have mode in %s (actual %o)' % 348 (crash_dir, ("%o" % m for m in permitted_modes), mode)) 349 350 351 def _check_minidump_stackwalk(self, minidump_path, basename, 352 from_crash_reporter): 353 stack = utils.system_output('/usr/bin/minidump_stackwalk %s %s' % 354 (minidump_path, self._symbol_dir)) 355 self._verify_stack(stack, basename, from_crash_reporter) 356 357 358 def _check_generated_report_sending(self, meta_path, payload_path, 359 exec_name, report_kind, 360 expected_sig=None): 361 # Now check that the sending works 362 result = self._call_sender_one_crash( 363 report=os.path.basename(payload_path)) 364 if (not result['send_attempt'] or not result['send_success'] or 365 result['report_exists']): 366 raise error.TestFail('Report not sent properly') 367 if result['exec_name'] != exec_name: 368 raise error.TestFail('Executable name incorrect') 369 if result['report_kind'] != report_kind: 370 raise error.TestFail('Expected a %s report' % report_kind) 371 if result['report_payload'] != payload_path: 372 raise error.TestFail('Sent the wrong minidump payload %s vs %s' % ( 373 result['report_payload'], payload_path)) 374 if result['meta_path'] != meta_path: 375 raise error.TestFail('Used the wrong meta file %s vs %s' % ( 376 result['meta_path'], meta_path)) 377 if expected_sig is None: 378 if result['sig'] is not None: 379 raise error.TestFail('Report should not have signature') 380 else: 381 if not 'sig' in result or result['sig'] != expected_sig: 382 raise error.TestFail('Report signature mismatch: %s vs %s' % 383 (result['sig'], expected_sig)) 384 385 version = self._expected_version 386 if version is None: 387 lsb_release = utils.read_file('/etc/lsb-release') 388 version = re.search( 389 r'CHROMEOS_RELEASE_VERSION=(.*)', lsb_release).group(1) 390 391 if not ('Version: %s' % version) in result['output']: 392 raise error.TestFail('Missing version %s in log output' % version) 393 394 395 def _run_crasher_process_and_analyze(self, username, 396 cause_crash=True, consent=True, 397 crasher_path=None, run_crasher=None, 398 expected_uid=None, expected_gid=None, 399 expected_exit_code=None, 400 expect_crash_reporter_fail=False): 401 self._log_reader.set_start_by_current() 402 403 result = self._run_crasher_process( 404 username, cause_crash=cause_crash, consent=consent, 405 crasher_path=crasher_path, run_crasher=run_crasher, 406 expected_uid=expected_uid, expected_gid=expected_gid, 407 expected_exit_code=expected_exit_code) 408 409 if not result['crashed'] or not result['crash_reporter_caught']: 410 return result 411 412 crash_dir = self._get_crash_dir(username, self._force_user_crash_dir) 413 crash_dir = self._canonicalize_crash_dir(crash_dir) 414 415 if not consent: 416 contents = os.listdir(crash_dir) 417 if contents: 418 raise error.TestFail( 419 'Crash directory should be empty but had %s', contents) 420 return result 421 422 if not os.path.exists(crash_dir): 423 raise error.TestFail('Crash directory does not exist') 424 425 crash_contents = os.listdir(crash_dir) 426 basename = os.path.basename(crasher_path or self._crasher_path) 427 if expect_crash_reporter_fail: 428 old_basename = basename 429 basename = "crash_reporter_failure" 430 431 # A dict tracking files for each crash report. 432 crash_report_files = {} 433 434 self._check_crash_directory_permissions(crash_dir) 435 436 logging.debug('Contents in %s: %s', crash_dir, crash_contents) 437 438 # Variables and their typical contents: 439 # basename: crasher_nobreakpad 440 # filename: crasher_nobreakpad.20181023.135339.12345.16890.dmp 441 # ext: dmp 442 for filename in crash_contents: 443 if filename.endswith('.core'): 444 # Ignore core files. We'll test them later. 445 pass 446 elif (expect_crash_reporter_fail 447 and filename.startswith(old_basename + '.')): 448 # In the case where crash reporter fails, we might generate 449 # some files with the basename of the crashing 450 # executable. That's okay -- just ignore them. 451 pass 452 elif filename.startswith(basename + '.'): 453 ext = filename.rsplit('.', 1)[1] 454 logging.debug('Found crash report file (%s): %s', ext, filename) 455 if ext in crash_report_files: 456 raise error.TestFail( 457 'Found multiple files with .%s: %s and %s' % 458 (ext, filename, crash_report_files[ext])) 459 crash_report_files[ext] = filename 460 else: 461 # Flag all unknown files. 462 raise error.TestFail('Crash reporter created an unknown file: ' 463 '%s' % (filename,)) 464 465 # Make sure we generated the exact set of files we expected. 466 found_filetypes = set(crash_report_files.keys()) 467 missing_filetypes = self.REPORT_REQUIRED_FILETYPES - found_filetypes 468 unknown_filetypes = (found_filetypes - self.REPORT_REQUIRED_FILETYPES - 469 self.REPORT_OPTIONAL_FILETYPES) 470 if missing_filetypes: 471 raise error.TestFail('crash report is missing files: %s' % ( 472 ['.' + x for x in missing_filetypes],)) 473 if unknown_filetypes: 474 raise error.TestFail('crash report includes unknown files: %s' % ( 475 [crash_report_files[x] for x in unknown_filetypes],)) 476 477 # Create full paths for the logging code below. 478 for key in (self.REPORT_REQUIRED_FILETYPES | 479 self.REPORT_OPTIONAL_FILETYPES): 480 if key in crash_report_files: 481 crash_report_files[key] = os.path.join( 482 crash_dir, crash_report_files[key]) 483 else: 484 crash_report_files[key] = None 485 486 result['minidump'] = crash_report_files['dmp'] 487 result['basename'] = basename 488 result['meta'] = crash_report_files['meta'] 489 result['log'] = crash_report_files['log'] 490 result['pslog'] = crash_report_files['pslog'] 491 return result 492 493 494 def _check_crashed_and_caught(self, result): 495 if not result['crashed']: 496 raise error.TestFail('Crasher returned %d instead of crashing' % 497 result['returncode']) 498 499 if not result['crash_reporter_caught']: 500 logging.debug('Logs do not contain crash_reporter message:\n%s', 501 self._log_reader.get_logs()) 502 raise error.TestFail('crash_reporter did not catch crash') 503 504 505 def _check_crashing_process(self, 506 username, 507 consent=True, 508 crasher_path=None, 509 run_crasher=None, 510 expected_uid=None, 511 expected_gid=None, 512 expected_exit_code=None, 513 extra_meta_contents=None): 514 result = self._run_crasher_process_and_analyze( 515 username, consent=consent, 516 crasher_path=crasher_path, 517 run_crasher=run_crasher, 518 expected_uid=expected_uid, 519 expected_gid=expected_gid, 520 expected_exit_code=expected_exit_code) 521 522 self._check_crashed_and_caught(result) 523 524 if not consent: 525 return 526 527 if extra_meta_contents: 528 with open(result['meta'], 'r') as f: 529 if extra_meta_contents not in f.read(): 530 raise error.TestFail('metadata did not contain "%s"' % 531 extra_meta_contents) 532 533 if not result['minidump']: 534 raise error.TestFail('crash reporter did not generate minidump') 535 536 if not self._log_reader.can_find('Stored minidump to ' + 537 result['minidump']): 538 raise error.TestFail('crash reporter did not announce minidump') 539 540 self._check_minidump_stackwalk(result['minidump'], 541 result['basename'], 542 from_crash_reporter=True) 543 self._check_generated_report_sending(result['meta'], 544 result['minidump'], 545 result['basename'], 546 'minidump') 547