1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import grp 6import logging 7import os 8import pwd 9import re 10import shutil 11import signal 12import stat 13import subprocess 14 15import crash_test 16from autotest_lib.client.bin import utils 17from autotest_lib.client.common_lib import error 18 19 20CRASHER = 'crasher_nobreakpad' 21 22 23class UserCrashTest(crash_test.CrashTest): 24 """ 25 Base class for tests that verify crash reporting for user processes. Shared 26 functionality includes installing a crasher executable, generating Breakpad 27 symbols, running the crasher process, and verifying collection and sending. 28 """ 29 30 31 # Every crash report needs one of these to be valid. 32 REPORT_REQUIRED_FILETYPES = {'meta'} 33 # Reports might have these and that's OK! 34 REPORT_OPTIONAL_FILETYPES = {'dmp', 'log', 'proclog'} 35 36 37 def setup(self): 38 """Copy the crasher source code under |srcdir| and build it.""" 39 src = os.path.join(os.path.dirname(__file__), 'crasher') 40 dest = os.path.join(self.srcdir, 'crasher') 41 shutil.copytree(src, dest) 42 43 os.chdir(dest) 44 utils.make() 45 46 47 def initialize(self, expected_tag='user', expected_version=None, 48 force_user_crash_dir=False): 49 """Initialize and configure the test. 50 51 @param expected_tag: Expected tag in crash_reporter log message. 52 @param expected_version: Expected version included in the crash report, 53 or None to use the Chrome OS version. 54 @param force_user_crash_dir: Always look for crash reports in the crash 55 directory of the current user session, or 56 the fallback directory if no sessions. 57 """ 58 crash_test.CrashTest.initialize(self) 59 self._expected_tag = expected_tag 60 self._expected_version = expected_version 61 self._force_user_crash_dir = force_user_crash_dir 62 63 64 def _prepare_crasher(self, root_path='/'): 65 """Extract the crasher and set its permissions. 66 67 crasher is only gzipped to subvert Portage stripping. 68 69 @param root_path: Root directory of the chroot environment in which the 70 crasher is installed and run. 71 """ 72 self._root_path = root_path 73 self._crasher_path = os.path.join(self.srcdir, 'crasher', CRASHER) 74 utils.system('cd %s; tar xzf crasher.tgz-unmasked' % 75 os.path.dirname(self._crasher_path)) 76 # Make sure all users (specifically chronos) have access to 77 # this directory and its decendents in order to run crasher 78 # executable as different users. 79 utils.system('chmod -R a+rx ' + self.bindir) 80 81 82 def _populate_symbols(self): 83 """Set up Breakpad's symbol structure. 84 85 Breakpad's minidump processor expects symbols to be in a directory 86 hierarchy: 87 <symbol-root>/<module_name>/<file_id>/<module_name>.sym 88 """ 89 self._symbol_dir = os.path.join(os.path.dirname(self._crasher_path), 90 'symbols') 91 utils.system('rm -rf %s' % self._symbol_dir) 92 os.mkdir(self._symbol_dir) 93 94 basename = os.path.basename(self._crasher_path) 95 utils.system('/usr/bin/dump_syms %s > %s.sym' % 96 (self._crasher_path, 97 basename)) 98 sym_name = '%s.sym' % basename 99 symbols = utils.read_file(sym_name) 100 # First line should be like: 101 # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX 102 # or 103 # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX 104 first_line = symbols.split('\n')[0] 105 tokens = first_line.split() 106 if tokens[0] != 'MODULE' or tokens[1] != 'Linux': 107 raise error.TestError('Unexpected symbols format: %s', 108 first_line) 109 file_id = tokens[3] 110 target_dir = os.path.join(self._symbol_dir, basename, file_id) 111 os.makedirs(target_dir) 112 os.rename(sym_name, os.path.join(target_dir, sym_name)) 113 114 115 def _is_frame_in_stack(self, frame_index, module_name, 116 function_name, file_name, 117 line_number, stack): 118 """Search for frame entries in the given stack dump text. 119 120 A frame entry looks like (alone on a line): 121 16 crasher_nobreakpad!main [crasher.cc : 21 + 0xb] 122 123 Args: 124 frame_index: number of the stack frame (0 is innermost frame) 125 module_name: name of the module (executable or dso) 126 function_name: name of the function in the stack 127 file_name: name of the file containing the function 128 line_number: line number 129 stack: text string of stack frame entries on separate lines. 130 131 Returns: 132 Boolean indicating if an exact match is present. 133 134 Note: 135 We do not care about the full function signature - ie, is it 136 foo or foo(ClassA *). These are present in function names 137 pulled by dump_syms for Stabs but not for DWARF. 138 """ 139 regexp = (r'\n\s*%d\s+%s!%s.*\[\s*%s\s*:\s*%d\s.*\]' % 140 (frame_index, module_name, 141 function_name, file_name, 142 line_number)) 143 logging.info('Searching for regexp %s', regexp) 144 return re.search(regexp, stack) is not None 145 146 147 def _verify_stack(self, stack, basename, from_crash_reporter): 148 # Should identify cause as SIGSEGV at address 0x16. 149 logging.debug('minidump_stackwalk output:\n%s', stack) 150 151 # Look for a line like: 152 # Crash reason: SIGSEGV 153 # Crash reason: SIGSEGV /0x00000000 154 match = re.search(r'Crash reason:\s+([^\s]*)', stack) 155 expected_address = '0x16' 156 if not match or match.group(1) != 'SIGSEGV': 157 raise error.TestFail('Did not identify SIGSEGV cause') 158 match = re.search(r'Crash address:\s+(.*)', stack) 159 if not match or match.group(1) != expected_address: 160 raise error.TestFail('Did not identify crash address %s' % 161 expected_address) 162 163 # Should identify crash at *(char*)0x16 assignment line 164 if not self._is_frame_in_stack(0, basename, 165 'recbomb', 'bomb.cc', 9, stack): 166 raise error.TestFail('Did not show crash line on stack') 167 168 # Should identify recursion line which is on the stack 169 # for 15 levels 170 if not self._is_frame_in_stack(15, basename, 'recbomb', 171 'bomb.cc', 12, stack): 172 raise error.TestFail('Did not show recursion line on stack') 173 174 # Should identify main line 175 if not self._is_frame_in_stack(16, basename, 'main', 176 'crasher.cc', 24, stack): 177 raise error.TestFail('Did not show main on stack') 178 179 180 def _run_crasher_process(self, username, cause_crash=True, consent=True, 181 crasher_path=None, run_crasher=None, 182 expected_uid=None, expected_gid=None, 183 expected_exit_code=None, expected_reason=None): 184 """Runs the crasher process. 185 186 Will wait up to 10 seconds for crash_reporter to report the crash. 187 crash_reporter_caught will be marked as true when the "Received crash 188 notification message..." appears. While associated logs are likely to be 189 available at this point, the function does not guarantee this. 190 191 @param username: Unix user of the crasher process. 192 @param cause_crash: Whether the crasher should crash. 193 @param consent: Whether the user consents to crash reporting. 194 @param crasher_path: Path to which the crasher should be copied before 195 execution. Relative to |_root_path|. 196 @param run_crasher: A closure to override the default |crasher_command| 197 invocation. It should return a tuple describing the 198 process, where |pid| can be None if it should be 199 parsed from the |output|: 200 201 def run_crasher(username, crasher_command): 202 ... 203 return (exit_code, output, pid) 204 205 @param expected_uid: The uid the crash happens under. 206 @param expected_gid: The gid the crash happens under. 207 @param expected_exit_code: 208 @param expected_reason: 209 Expected information in crash_reporter log message. 210 211 @returns: 212 A dictionary with keys: 213 returncode: return code of the crasher 214 crashed: did the crasher return segv error code 215 crash_reporter_caught: did crash_reporter catch a segv 216 output: stderr output of the crasher process 217 """ 218 if crasher_path is None: 219 crasher_path = self._crasher_path 220 else: 221 dest = os.path.join(self._root_path, 222 crasher_path[os.path.isabs(crasher_path):]) 223 224 utils.system('cp -a "%s" "%s"' % (self._crasher_path, dest)) 225 226 self.enable_crash_filtering(os.path.basename(crasher_path)) 227 228 crasher_command = [] 229 230 if username == 'root': 231 if expected_exit_code is None: 232 expected_exit_code = -signal.SIGSEGV 233 else: 234 if expected_exit_code is None: 235 expected_exit_code = 128 + signal.SIGSEGV 236 237 if not run_crasher: 238 crasher_command.extend(['su', username, '-c']) 239 240 crasher_command.append(crasher_path) 241 basename = os.path.basename(crasher_path) 242 if not cause_crash: 243 crasher_command.append('--nocrash') 244 self._set_consent(consent) 245 246 logging.debug('Running crasher: %s', crasher_command) 247 248 if run_crasher: 249 (exit_code, output, pid) = run_crasher(username, crasher_command) 250 251 else: 252 crasher = subprocess.Popen(crasher_command, 253 stdout=subprocess.PIPE, 254 stderr=subprocess.PIPE) 255 256 output = crasher.communicate()[1] 257 exit_code = crasher.returncode 258 pid = None 259 260 logging.debug('Crasher output:\n%s', output) 261 262 if pid is None: 263 # Get the PID from the output, since |crasher.pid| may be su's PID. 264 match = re.search(r'pid=(\d+)', output) 265 if not match: 266 raise error.TestFail('Missing PID in crasher output') 267 pid = int(match.group(1)) 268 269 if expected_uid is None: 270 expected_uid = pwd.getpwnam(username).pw_uid 271 272 if expected_gid is None: 273 expected_gid = pwd.getpwnam(username).pw_gid 274 275 if expected_reason is None: 276 expected_reason = 'handling' if consent else 'ignoring - no consent' 277 278 expected_message = ( 279 ('[%s] Received crash notification for %s[%d] sig 11, user %d ' 280 'group %d (%s)') % 281 (self._expected_tag, basename, pid, expected_uid, expected_gid, 282 expected_reason)) 283 284 # Wait until no crash_reporter is running. 285 utils.poll_for_condition( 286 lambda: utils.system('pgrep -f crash_reporter.*:%s' % basename, 287 ignore_status=True) != 0, 288 timeout=10, 289 exception=error.TestError( 290 'Timeout waiting for crash_reporter to finish: ' + 291 self._log_reader.get_logs())) 292 293 is_caught = False 294 try: 295 utils.poll_for_condition( 296 lambda: self._log_reader.can_find(expected_message), 297 timeout=5, 298 desc='Logs contain crash_reporter message: ' + expected_message) 299 is_caught = True 300 except utils.TimeoutError: 301 pass 302 303 result = {'crashed': exit_code == expected_exit_code, 304 'crash_reporter_caught': is_caught, 305 'output': output, 306 'returncode': exit_code} 307 logging.debug('Crasher process result: %s', result) 308 return result 309 310 311 def _check_crash_directory_permissions(self, crash_dir): 312 stat_info = os.stat(crash_dir) 313 user = pwd.getpwuid(stat_info.st_uid).pw_name 314 group = grp.getgrgid(stat_info.st_gid).gr_name 315 mode = stat.S_IMODE(stat_info.st_mode) 316 317 if crash_dir.startswith('/var/spool/crash'): 318 if stat.S_ISDIR(stat_info.st_mode): 319 utils.system('ls -l %s' % crash_dir) 320 for f in os.listdir(crash_dir): 321 self._check_crash_directory_permissions( 322 os.path.join(crash_dir, f)) 323 permitted_modes = set([0o2770]) 324 else: 325 permitted_modes = set([0o660, 0o640, 0o644]) 326 expected_user = 'root' 327 expected_group = 'crash-access' 328 else: 329 permitted_modes = set([0o2770]) 330 expected_user = 'chronos' 331 expected_group = 'crash-user-access' 332 333 if user != expected_user or group != expected_group: 334 raise error.TestFail( 335 'Expected %s.%s ownership of %s (actual %s.%s)' % 336 (expected_user, expected_group, crash_dir, user, group)) 337 if mode not in permitted_modes: 338 raise error.TestFail( 339 'Expected %s to have mode in %s (actual %o)' % 340 (crash_dir, ("%o" % m for m in permitted_modes), mode)) 341 342 343 def _check_minidump_stackwalk(self, minidump_path, basename, 344 from_crash_reporter): 345 stack = utils.system_output('/usr/bin/minidump_stackwalk %s %s' % 346 (minidump_path, self._symbol_dir)) 347 self._verify_stack(stack, basename, from_crash_reporter) 348 349 350 def _check_generated_report_sending(self, meta_path, payload_path, 351 exec_name, report_kind, 352 expected_sig=None): 353 # Now check that the sending works 354 result = self._call_sender_one_crash( 355 report=os.path.basename(payload_path)) 356 if (not result['send_attempt'] or not result['send_success'] or 357 result['report_exists']): 358 raise error.TestFail('Report not sent properly') 359 if result['exec_name'] != exec_name: 360 raise error.TestFail('Executable name incorrect') 361 if result['report_kind'] != report_kind: 362 raise error.TestFail('Expected a %s report' % report_kind) 363 if result['report_payload'] != payload_path: 364 raise error.TestFail('Sent the wrong minidump payload %s vs %s' % ( 365 result['report_payload'], payload_path)) 366 if result['meta_path'] != meta_path: 367 raise error.TestFail('Used the wrong meta file %s vs %s' % ( 368 result['meta_path'], meta_path)) 369 if expected_sig is None: 370 if result['sig'] is not None: 371 raise error.TestFail('Report should not have signature') 372 else: 373 if not 'sig' in result or result['sig'] != expected_sig: 374 raise error.TestFail('Report signature mismatch: %s vs %s' % 375 (result['sig'], expected_sig)) 376 377 version = self._expected_version 378 if version is None: 379 lsb_release = utils.read_file('/etc/lsb-release') 380 version = re.search( 381 r'CHROMEOS_RELEASE_VERSION=(.*)', lsb_release).group(1) 382 383 if not ('Version: %s' % version) in result['output']: 384 raise error.TestFail('Missing version %s in log output' % version) 385 386 387 def _run_crasher_process_and_analyze(self, username, 388 cause_crash=True, consent=True, 389 crasher_path=None, run_crasher=None, 390 expected_uid=None, expected_gid=None, 391 expected_exit_code=None): 392 self._log_reader.set_start_by_current() 393 394 result = self._run_crasher_process( 395 username, cause_crash=cause_crash, consent=consent, 396 crasher_path=crasher_path, run_crasher=run_crasher, 397 expected_uid=expected_uid, expected_gid=expected_gid, 398 expected_exit_code=expected_exit_code) 399 400 if not result['crashed'] or not result['crash_reporter_caught']: 401 return result 402 403 crash_dir = self._get_crash_dir(username, self._force_user_crash_dir) 404 crash_dir = self._canonicalize_crash_dir(crash_dir) 405 406 if not consent: 407 if os.path.exists(crash_dir): 408 raise error.TestFail('Crash directory should not exist') 409 return result 410 411 if not os.path.exists(crash_dir): 412 raise error.TestFail('Crash directory does not exist') 413 414 crash_contents = os.listdir(crash_dir) 415 basename = os.path.basename(crasher_path or self._crasher_path) 416 417 # A dict tracking files for each crash report. 418 crash_report_files = {} 419 420 self._check_crash_directory_permissions(crash_dir) 421 422 logging.debug('Contents in %s: %s', crash_dir, crash_contents) 423 424 # Variables and their typical contents: 425 # basename: crasher_nobreakpad 426 # filename: crasher_nobreakpad.20181023.135339.16890.dmp 427 # ext: dmp 428 for filename in crash_contents: 429 if filename.endswith('.core'): 430 # Ignore core files. We'll test them later. 431 pass 432 elif filename.startswith(basename + '.'): 433 ext = filename.rsplit('.', 1)[1] 434 logging.debug('Found crash report file (%s): %s', ext, filename) 435 if ext in crash_report_files: 436 raise error.TestFail( 437 'Found multiple files with .%s: %s and %s' % 438 (ext, filename, crash_report_files[ext])) 439 crash_report_files[ext] = filename 440 else: 441 # Flag all unknown files. 442 raise error.TestFail('Crash reporter created an unknown file: ' 443 '%s' % (filename,)) 444 445 # Make sure we generated the exact set of files we expected. 446 found_filetypes = set(crash_report_files.keys()) 447 missing_filetypes = self.REPORT_REQUIRED_FILETYPES - found_filetypes 448 unknown_filetypes = (found_filetypes - self.REPORT_REQUIRED_FILETYPES - 449 self.REPORT_OPTIONAL_FILETYPES) 450 if missing_filetypes: 451 raise error.TestFail('crash report is missing files: %s' % ( 452 ['.' + x for x in missing_filetypes],)) 453 if unknown_filetypes: 454 raise error.TestFail('crash report includes unknown files: %s' % ( 455 [crash_report_files[x] for x in unknown_filetypes],)) 456 457 # Create full paths for the logging code below. 458 for key in (self.REPORT_REQUIRED_FILETYPES | 459 self.REPORT_OPTIONAL_FILETYPES): 460 if key in crash_report_files: 461 crash_report_files[key] = os.path.join( 462 crash_dir, crash_report_files[key]) 463 else: 464 crash_report_files[key] = None 465 466 result['minidump'] = crash_report_files['dmp'] 467 result['basename'] = basename 468 result['meta'] = crash_report_files['meta'] 469 result['log'] = crash_report_files['log'] 470 return result 471 472 473 def _check_crashed_and_caught(self, result): 474 if not result['crashed']: 475 raise error.TestFail('Crasher returned %d instead of crashing' % 476 result['returncode']) 477 478 if not result['crash_reporter_caught']: 479 logging.debug('Logs do not contain crash_reporter message:\n%s', 480 self._log_reader.get_logs()) 481 raise error.TestFail('crash_reporter did not catch crash') 482 483 484 def _check_crashing_process(self, username, consent=True, 485 crasher_path=None, run_crasher=None, 486 expected_uid=None, expected_gid=None, 487 expected_exit_code=None): 488 result = self._run_crasher_process_and_analyze( 489 username, consent=consent, 490 crasher_path=crasher_path, 491 run_crasher=run_crasher, 492 expected_uid=expected_uid, 493 expected_gid=expected_gid, 494 expected_exit_code=expected_exit_code) 495 496 self._check_crashed_and_caught(result) 497 498 if not consent: 499 return 500 501 if not result['minidump']: 502 raise error.TestFail('crash reporter did not generate minidump') 503 504 if not self._log_reader.can_find('Stored minidump to ' + 505 result['minidump']): 506 raise error.TestFail('crash reporter did not announce minidump') 507 508 self._check_minidump_stackwalk(result['minidump'], 509 result['basename'], 510 from_crash_reporter=True) 511 self._check_generated_report_sending(result['meta'], 512 result['minidump'], 513 result['basename'], 514 'minidump') 515