1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import grp 6import logging 7import os 8import pwd 9import re 10import shutil 11import signal 12import stat 13import subprocess 14 15from autotest_lib.client.cros.crash import crash_test 16from autotest_lib.client.bin import utils 17from autotest_lib.client.common_lib import error 18 19 20CRASHER = 'crasher_nobreakpad' 21 22 23class UserCrashTest(crash_test.CrashTest): 24 """ 25 Base class for tests that verify crash reporting for user processes. Shared 26 functionality includes installing a crasher executable, generating Breakpad 27 symbols, running the crasher process, and verifying collection and sending. 28 """ 29 30 31 # Every crash report needs one of these to be valid. 32 REPORT_REQUIRED_FILETYPES = {'meta'} 33 # Reports might have these and that's OK! 34 REPORT_OPTIONAL_FILETYPES = {'dmp', 'log', 'proclog', 'pslog'} 35 36 37 def setup(self): 38 """Copy the crasher source code under |srcdir| and build it.""" 39 src = os.path.join(os.path.dirname(__file__), 'crasher') 40 dest = os.path.join(self.srcdir, 'crasher') 41 shutil.copytree(src, dest) 42 43 os.chdir(dest) 44 utils.make() 45 46 47 def initialize(self, expected_tag='user', expected_version=None, 48 force_user_crash_dir=False): 49 """Initialize and configure the test. 50 51 @param expected_tag: Expected tag in crash_reporter log message. 52 @param expected_version: Expected version included in the crash report, 53 or None to use the ChromeOS version. 54 @param force_user_crash_dir: Always look for crash reports in the crash 55 directory of the current user session, or 56 the fallback directory if no sessions. 57 """ 58 crash_test.CrashTest.initialize(self) 59 self._expected_tag = expected_tag 60 self._expected_version = expected_version 61 self._force_user_crash_dir = force_user_crash_dir 62 63 64 def _prepare_crasher(self, root_path='/'): 65 """Extract the crasher and set its permissions. 66 67 crasher is only gzipped to subvert Portage stripping. 68 69 @param root_path: Root directory of the chroot environment in which the 70 crasher is installed and run. 71 """ 72 self._root_path = root_path 73 self._crasher_path = os.path.join(self.srcdir, 'crasher', CRASHER) 74 utils.system('cd %s; tar xzf crasher.tgz-unmasked' % 75 os.path.dirname(self._crasher_path)) 76 # Make sure all users (specifically chronos) have access to 77 # this directory and its decendents in order to run crasher 78 # executable as different users. 79 utils.system('chmod -R a+rx ' + self.bindir) 80 81 82 def _populate_symbols(self): 83 """Set up Breakpad's symbol structure. 84 85 Breakpad's minidump processor expects symbols to be in a directory 86 hierarchy: 87 <symbol-root>/<module_name>/<file_id>/<module_name>.sym 88 """ 89 self._symbol_dir = os.path.join(os.path.dirname(self._crasher_path), 90 'symbols') 91 utils.system('rm -rf %s' % self._symbol_dir) 92 os.mkdir(self._symbol_dir) 93 94 basename = os.path.basename(self._crasher_path) 95 utils.system('dump_syms %s > %s.sym' % (self._crasher_path, basename)) 96 sym_name = '%s.sym' % basename 97 symbols = utils.read_file(sym_name) 98 # First line should be like: 99 # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX 100 # or 101 # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX 102 first_line = symbols.split('\n')[0] 103 tokens = first_line.split() 104 if tokens[0] != 'MODULE' or tokens[1] != 'Linux': 105 raise error.TestError('Unexpected symbols format: %s', first_line) 106 file_id = tokens[3] 107 target_dir = os.path.join(self._symbol_dir, basename, file_id) 108 os.makedirs(target_dir) 109 os.rename(sym_name, os.path.join(target_dir, sym_name)) 110 111 112 def _is_frame_in_stack(self, frame_index, module_name, 113 function_name, file_name, 114 line_number, stack): 115 """Search for frame entries in the given stack dump text. 116 117 A frame entry looks like (alone on a line): 118 16 crasher_nobreakpad!main [crasher.cc : 21 + 0xb] 119 120 Args: 121 frame_index: number of the stack frame (0 is innermost frame) 122 module_name: name of the module (executable or dso) 123 function_name: name of the function in the stack 124 file_name: name of the file containing the function 125 line_number: line number 126 stack: text string of stack frame entries on separate lines. 127 128 Returns: 129 Boolean indicating if an exact match is present. 130 131 Note: 132 We do not care about the full function signature - ie, is it 133 foo or foo(ClassA *). These are present in function names 134 pulled by dump_syms for Stabs but not for DWARF. 135 """ 136 regexp = (r'\n\s*%d\s+%s!%s.*\[\s*%s\s*:\s*%d\s.*\]' % 137 (frame_index, module_name, 138 function_name, file_name, 139 line_number)) 140 logging.info('Searching for regexp %s', regexp) 141 return re.search(regexp, stack) is not None 142 143 144 def _verify_stack(self, stack, basename, from_crash_reporter): 145 # Should identify cause as SIGSEGV at address 0x16. 146 logging.debug('minidump_stackwalk output:\n%s', stack) 147 148 # Look for a line like: 149 # Crash reason: SIGSEGV 150 # Crash reason: SIGSEGV /0x00000000 151 match = re.search(r'Crash reason:\s+([^\s]*)', stack) 152 expected_address = '0x16' 153 if not match or match.group(1) != 'SIGSEGV': 154 raise error.TestFail('Did not identify SIGSEGV cause') 155 match = re.search(r'Crash address:\s+(.*)', stack) 156 if not match or match.group(1) != expected_address: 157 raise error.TestFail('Did not identify crash address %s' % 158 expected_address) 159 160 # Should identify crash at *(char*)0x16 assignment line 161 if not self._is_frame_in_stack(0, basename, 162 'recbomb', 'bomb.cc', 9, stack): 163 raise error.TestFail('Did not show crash line on stack') 164 165 # Should identify recursion line which is on the stack 166 # for 15 levels 167 if not self._is_frame_in_stack(15, basename, 'recbomb', 168 'bomb.cc', 12, stack): 169 raise error.TestFail('Did not show recursion line on stack') 170 171 # Should identify main line 172 if not self._is_frame_in_stack(16, basename, 'main', 173 'crasher.cc', 24, stack): 174 raise error.TestFail('Did not show main on stack') 175 176 177 def _run_crasher_process(self, username, cause_crash=True, consent=True, 178 crasher_path=None, run_crasher=None, 179 expected_uid=None, expected_gid=None, 180 expected_exit_code=None, expected_reason=None): 181 """Runs the crasher process. 182 183 Will wait up to 10 seconds for crash_reporter to report the crash. 184 185 @param username: Unix user of the crasher process. 186 @param cause_crash: Whether the crasher should crash. 187 @param consent: Whether the user consents to crash reporting. 188 @param crasher_path: Path to which the crasher should be copied before 189 execution. Relative to |_root_path|. 190 @param run_crasher: A closure to override the default |crasher_command| 191 invocation. It should return a tuple describing the 192 process, where |pid| can be None if it should be 193 parsed from the |output|: 194 195 def run_crasher(username, crasher_command): 196 ... 197 return (exit_code, output, pid) 198 199 @param expected_uid: The uid the crash happens under. 200 @param expected_gid: The gid the crash happens under. 201 @param expected_exit_code: 202 @param expected_reason: 203 Expected information in crash_reporter log message. 204 205 @returns: 206 A dictionary with keys: 207 returncode: return code of the crasher 208 crashed: did the crasher return segv error code 209 output: stderr output of the crasher process 210 """ 211 if crasher_path is None: 212 crasher_path = self._crasher_path 213 else: 214 dest = os.path.join(self._root_path, 215 crasher_path[os.path.isabs(crasher_path):]) 216 217 utils.system('cp -a "%s" "%s"' % (self._crasher_path, dest)) 218 219 # Limit to the first 15 characters of the crasher binary name because 220 # that's what the kernel invokes crash_reporter with. 221 self.enable_crash_filtering(os.path.basename(crasher_path)[:15]) 222 223 crasher_command = [] 224 225 if username == 'root': 226 if expected_exit_code is None: 227 expected_exit_code = -signal.SIGSEGV 228 else: 229 if expected_exit_code is None: 230 expected_exit_code = 128 + signal.SIGSEGV 231 232 if not run_crasher: 233 crasher_command.extend(['su', username, '-c']) 234 235 crasher_command.append(crasher_path) 236 basename = os.path.basename(crasher_path) 237 if not cause_crash: 238 crasher_command.append('--nocrash') 239 self._set_consent(consent) 240 241 logging.debug('Running crasher: %s', crasher_command) 242 243 if run_crasher: 244 (exit_code, output, pid) = run_crasher(username, crasher_command) 245 246 else: 247 crasher = subprocess.Popen(crasher_command, 248 stdout=subprocess.PIPE, 249 stderr=subprocess.PIPE) 250 251 output = crasher.communicate()[1].decode() 252 exit_code = crasher.returncode 253 pid = None 254 255 logging.debug('Crasher output:\n%s', output) 256 257 if pid is None: 258 # Get the PID from the output, since |crasher.pid| may be su's PID. 259 match = re.search(r'pid=(\d+)', output) 260 if not match: 261 raise error.TestFail('Missing PID in crasher output') 262 pid = int(match.group(1)) 263 264 if expected_uid is None: 265 expected_uid = pwd.getpwnam(username).pw_uid 266 267 if expected_gid is None: 268 expected_gid = pwd.getpwnam(username).pw_gid 269 270 if expected_reason is None and consent: 271 expected_reason = 'handling' 272 273 if expected_reason is not None: 274 expected_message = (( 275 '[%s] Received crash notification for %s[%d] sig 11, user %d ' 276 'group %d (%s)') % 277 (self._expected_tag, basename, pid, 278 expected_uid, expected_gid, expected_reason)) 279 else: 280 # No consent; different message format. 281 expected_message = (( 282 'No consent. Not handling invocation: /sbin/crash_reporter ' 283 '--user=%d:11:%d:%d:%s') % 284 (pid, expected_uid, expected_gid, basename)) 285 286 # Wait until no crash_reporter is running. 287 utils.poll_for_condition( 288 lambda: utils.system('pgrep -f crash_reporter.*:%s' % basename, 289 ignore_status=True) != 0, 290 timeout=10, 291 exception=error.TestError( 292 'Timeout waiting for crash_reporter to finish: ' + 293 self._log_reader.get_logs())) 294 295 result = {'crashed': exit_code == expected_exit_code, 296 'output': output, 297 'returncode': exit_code} 298 logging.debug('Crasher process result: %s', result) 299 return result 300 301 302 def _check_crash_directory_permissions(self, crash_dir): 303 stat_info = os.stat(crash_dir) 304 user = pwd.getpwuid(stat_info.st_uid).pw_name 305 group = grp.getgrgid(stat_info.st_gid).gr_name 306 mode = stat.S_IMODE(stat_info.st_mode) 307 308 if crash_dir.startswith('/var/spool/crash'): 309 if stat.S_ISDIR(stat_info.st_mode): 310 utils.system('ls -l %s' % crash_dir) 311 for f in os.listdir(crash_dir): 312 self._check_crash_directory_permissions( 313 os.path.join(crash_dir, f)) 314 permitted_modes = set([0o2770]) 315 else: 316 permitted_modes = set([0o660, 0o640, 0o644]) 317 expected_user = 'root' 318 expected_group = 'crash-access' 319 else: 320 permitted_modes = set([0o2770]) 321 expected_user = 'chronos' 322 expected_group = 'crash-user-access' 323 324 if user != expected_user or group != expected_group: 325 raise error.TestFail( 326 'Expected %s.%s ownership of %s (actual %s.%s)' % 327 (expected_user, expected_group, crash_dir, user, group)) 328 if mode not in permitted_modes: 329 raise error.TestFail( 330 'Expected %s to have mode in %s (actual %o)' % 331 (crash_dir, ("%o" % m for m in permitted_modes), mode)) 332 333 334 def _check_minidump_stackwalk(self, minidump_path, basename, 335 from_crash_reporter): 336 stack = utils.system_output('minidump_stackwalk %s %s' % 337 (minidump_path, self._symbol_dir)) 338 self._verify_stack(stack, basename, from_crash_reporter) 339 340 341 def _check_generated_report_sending(self, meta_path, payload_path, 342 exec_name, report_kind, 343 expected_sig=None): 344 # Now check that the sending works 345 result = self._call_sender_one_crash( 346 report=os.path.basename(payload_path)) 347 if (not result['send_attempt'] or not result['send_success'] or 348 result['report_exists']): 349 raise error.TestFail('Report not sent properly') 350 if result['exec_name'] != exec_name: 351 raise error.TestFail('Executable name incorrect') 352 if result['report_kind'] != report_kind: 353 raise error.TestFail('Expected a %s report' % report_kind) 354 if result['report_payload'] != payload_path: 355 raise error.TestFail('Sent the wrong minidump payload %s vs %s' % ( 356 result['report_payload'], payload_path)) 357 if result['meta_path'] != meta_path: 358 raise error.TestFail('Used the wrong meta file %s vs %s' % ( 359 result['meta_path'], meta_path)) 360 if expected_sig is None: 361 if result['sig'] is not None: 362 raise error.TestFail('Report should not have signature') 363 else: 364 if not 'sig' in result or result['sig'] != expected_sig: 365 raise error.TestFail('Report signature mismatch: %s vs %s' % 366 (result['sig'], expected_sig)) 367 368 version = self._expected_version 369 if version is None: 370 lsb_release = utils.read_file('/etc/lsb-release') 371 version = re.search( 372 r'CHROMEOS_RELEASE_VERSION=(.*)', lsb_release).group(1) 373 374 if not ('Version: %s' % version) in result['output']: 375 raise error.TestFail('Missing version %s in log output' % version) 376 377 378 def _run_crasher_process_and_analyze(self, username, 379 cause_crash=True, consent=True, 380 crasher_path=None, run_crasher=None, 381 expected_uid=None, expected_gid=None, 382 expected_exit_code=None, 383 expect_crash_reporter_fail=False): 384 self._log_reader.set_start_by_current() 385 386 result = self._run_crasher_process( 387 username, cause_crash=cause_crash, consent=consent, 388 crasher_path=crasher_path, run_crasher=run_crasher, 389 expected_uid=expected_uid, expected_gid=expected_gid, 390 expected_exit_code=expected_exit_code) 391 392 if not result['crashed']: 393 return result 394 395 crash_dir = self._get_crash_dir(username, self._force_user_crash_dir) 396 crash_dir = self._canonicalize_crash_dir(crash_dir) 397 398 if not consent: 399 contents = os.listdir(crash_dir) 400 if contents: 401 raise error.TestFail( 402 'Crash directory should be empty but had %s', contents) 403 return result 404 405 if not os.path.exists(crash_dir): 406 raise error.TestFail('Crash directory does not exist') 407 408 crash_contents = os.listdir(crash_dir) 409 basename = os.path.basename(crasher_path or self._crasher_path) 410 if expect_crash_reporter_fail: 411 old_basename = basename 412 basename = "crash_reporter_failure" 413 414 # A dict tracking files for each crash report. 415 crash_report_files = {} 416 417 self._check_crash_directory_permissions(crash_dir) 418 419 logging.debug('Contents in %s: %s', crash_dir, crash_contents) 420 421 # Variables and their typical contents: 422 # basename: crasher_nobreakpad 423 # filename: crasher_nobreakpad.20181023.135339.12345.16890.dmp 424 # ext: dmp 425 for filename in crash_contents: 426 if filename.endswith('.core'): 427 # Ignore core files. We'll test them later. 428 pass 429 elif (expect_crash_reporter_fail 430 and filename.startswith(old_basename + '.')): 431 # In the case where crash reporter fails, we might generate 432 # some files with the basename of the crashing 433 # executable. That's okay -- just ignore them. 434 pass 435 elif filename.startswith(basename + '.'): 436 ext = filename.rsplit('.', 1)[1] 437 logging.debug('Found crash report file (%s): %s', ext, filename) 438 if ext in crash_report_files: 439 raise error.TestFail( 440 'Found multiple files with .%s: %s and %s' % 441 (ext, filename, crash_report_files[ext])) 442 crash_report_files[ext] = filename 443 else: 444 # Flag all unknown files. 445 raise error.TestFail('Crash reporter created an unknown file: ' 446 '%s' % (filename,)) 447 448 # Make sure we generated the exact set of files we expected. 449 found_filetypes = set(crash_report_files.keys()) 450 missing_filetypes = self.REPORT_REQUIRED_FILETYPES - found_filetypes 451 unknown_filetypes = (found_filetypes - self.REPORT_REQUIRED_FILETYPES - 452 self.REPORT_OPTIONAL_FILETYPES) 453 if missing_filetypes: 454 raise error.TestFail('crash report is missing files: %s' % ( 455 ['.' + x for x in missing_filetypes],)) 456 if unknown_filetypes: 457 raise error.TestFail('crash report includes unknown files: %s' % ( 458 [crash_report_files[x] for x in unknown_filetypes],)) 459 460 # Create full paths for the logging code below. 461 for key in (self.REPORT_REQUIRED_FILETYPES | 462 self.REPORT_OPTIONAL_FILETYPES): 463 if key in crash_report_files: 464 crash_report_files[key] = os.path.join( 465 crash_dir, crash_report_files[key]) 466 else: 467 crash_report_files[key] = None 468 469 result['minidump'] = crash_report_files['dmp'] 470 result['basename'] = basename 471 result['meta'] = crash_report_files['meta'] 472 result['log'] = crash_report_files['log'] 473 result['pslog'] = crash_report_files['pslog'] 474 return result 475 476 477 def _check_crashing_process(self, 478 username, 479 consent=True, 480 crasher_path=None, 481 run_crasher=None, 482 expected_uid=None, 483 expected_gid=None, 484 expected_exit_code=None, 485 extra_meta_contents=None): 486 result = self._run_crasher_process_and_analyze( 487 username, consent=consent, 488 crasher_path=crasher_path, 489 run_crasher=run_crasher, 490 expected_uid=expected_uid, 491 expected_gid=expected_gid, 492 expected_exit_code=expected_exit_code) 493 494 if not result['crashed']: 495 raise error.TestFail('Crasher returned %d instead of crashing' % 496 result['returncode']) 497 if not consent: 498 return 499 500 if extra_meta_contents: 501 with open(result['meta'], 'r') as f: 502 if extra_meta_contents not in f.read(): 503 raise error.TestFail('metadata did not contain "%s"' % 504 extra_meta_contents) 505 506 if not result['minidump']: 507 raise error.TestFail('crash reporter did not generate minidump') 508 509 self._check_minidump_stackwalk(result['minidump'], 510 result['basename'], 511 from_crash_reporter=True) 512 self._check_generated_report_sending(result['meta'], 513 result['minidump'], 514 result['basename'], 515 'minidump') 516