1#!/usr/bin/env python3 2# 3# Copyright (C) 2016 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import abc 18import argparse 19import filecmp 20import os 21import shlex 22import shutil 23import subprocess 24import sys 25 26from glob import glob 27from subprocess import DEVNULL 28from tempfile import mkdtemp 29 30sys.path.append(os.path.dirname(os.path.dirname( 31 os.path.realpath(__file__)))) 32 33from common.common import RetCode 34from common.common import CommandListToCommandString 35from common.common import FatalError 36from common.common import GetEnvVariableOrError 37from common.common import RunCommand 38from common.common import RunCommandForOutput 39from common.common import DeviceTestEnv 40 41# Return codes supported by bisection bug search. 42BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT) 43 44 45def GetExecutionModeRunner(dexer, debug_info, device, mode): 46 """Returns a runner for the given execution mode. 47 48 Args: 49 dexer: string, defines dexer 50 debug_info: boolean, if True include debugging info 51 device: string, target device serial number (or None) 52 mode: string, execution mode 53 Returns: 54 TestRunner with given execution mode 55 Raises: 56 FatalError: error for unknown execution mode 57 """ 58 if mode == 'ri': 59 return TestRunnerRIOnHost(debug_info) 60 if mode == 'hint': 61 return TestRunnerArtIntOnHost(dexer, debug_info) 62 if mode == 'hopt': 63 return TestRunnerArtOptOnHost(dexer, debug_info) 64 if mode == 'tint': 65 return TestRunnerArtIntOnTarget(dexer, debug_info, device) 66 if mode == 'topt': 67 return TestRunnerArtOptOnTarget(dexer, debug_info, device) 68 raise FatalError('Unknown execution mode') 69 70 71# 72# Execution mode classes. 73# 74 75 76class TestRunner(object): 77 """Abstraction for running a test in a particular execution mode.""" 78 __meta_class__ = abc.ABCMeta 79 80 @abc.abstractproperty 81 def description(self): 82 """Returns a description string of the execution mode.""" 83 84 @abc.abstractproperty 85 def id(self): 86 """Returns a short string that uniquely identifies the execution mode.""" 87 88 @property 89 def output_file(self): 90 return self.id + '_out.txt' 91 92 @abc.abstractmethod 93 def GetBisectionSearchArgs(self): 94 """Get arguments to pass to bisection search tool. 95 96 Returns: 97 list of strings - arguments for bisection search tool, or None if 98 runner is not bisectable 99 """ 100 101 @abc.abstractmethod 102 def CompileAndRunTest(self): 103 """Compile and run the generated test. 104 105 Ensures that the current Test.java in the temporary directory is compiled 106 and executed under the current execution mode. On success, transfers the 107 generated output to the file self.output_file in the temporary directory. 108 109 Most nonzero return codes are assumed non-divergent, since systems may 110 exit in different ways. This is enforced by normalizing return codes. 111 112 Returns: 113 normalized return code 114 """ 115 116 117class TestRunnerWithHostCompilation(TestRunner): 118 """Abstract test runner that supports compilation on host.""" 119 120 def __init__(self, dexer, debug_info): 121 """Constructor for the runner with host compilation. 122 123 Args: 124 dexer: string, defines dexer 125 debug_info: boolean, if True include debugging info 126 """ 127 self._dexer = dexer 128 self._debug_info = debug_info 129 130 def CompileOnHost(self): 131 if self._dexer == 'dx' or self._dexer == 'd8': 132 dbg = '-g' if self._debug_info else '-g:none' 133 if RunCommand(['javac', '--release=8', dbg, 'Test.java'], 134 out=None, err=None, timeout=30) == RetCode.SUCCESS: 135 dx = 'dx' if self._dexer == 'dx' else 'd8-compat-dx' 136 retc = RunCommand([dx, '--dex', '--output=classes.dex'] + glob('*.class'), 137 out=None, err='dxerr.txt', timeout=30) 138 else: 139 retc = RetCode.NOTCOMPILED 140 else: 141 raise FatalError('Unknown dexer: ' + self._dexer) 142 return retc 143 144 145class TestRunnerRIOnHost(TestRunner): 146 """Concrete test runner of the reference implementation on host.""" 147 148 def __init__(self, debug_info): 149 """Constructor for the runner with host compilation. 150 151 Args: 152 debug_info: boolean, if True include debugging info 153 """ 154 self._debug_info = debug_info 155 156 @property 157 def description(self): 158 return 'RI on host' 159 160 @property 161 def id(self): 162 return 'RI' 163 164 def CompileAndRunTest(self): 165 dbg = '-g' if self._debug_info else '-g:none' 166 if RunCommand(['javac', '--release=8', dbg, 'Test.java'], 167 out=None, err=None, timeout=30) == RetCode.SUCCESS: 168 retc = RunCommand(['java', 'Test'], self.output_file, err=None) 169 else: 170 retc = RetCode.NOTCOMPILED 171 return retc 172 173 def GetBisectionSearchArgs(self): 174 return None 175 176 177class TestRunnerArtOnHost(TestRunnerWithHostCompilation): 178 """Abstract test runner of Art on host.""" 179 180 def __init__(self, dexer, debug_info, extra_args=None): 181 """Constructor for the Art on host tester. 182 183 Args: 184 dexer: string, defines dexer 185 debug_info: boolean, if True include debugging info 186 extra_args: list of strings, extra arguments for dalvikvm 187 """ 188 super().__init__(dexer, debug_info) 189 self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex'] 190 if extra_args is not None: 191 self._art_cmd += extra_args 192 self._art_cmd.append('Test') 193 194 def CompileAndRunTest(self): 195 if self.CompileOnHost() == RetCode.SUCCESS: 196 retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt') 197 else: 198 retc = RetCode.NOTCOMPILED 199 return retc 200 201 202class TestRunnerArtIntOnHost(TestRunnerArtOnHost): 203 """Concrete test runner of interpreter mode Art on host.""" 204 205 def __init__(self, dexer, debug_info): 206 """Constructor for the Art on host tester (interpreter). 207 208 Args: 209 dexer: string, defines dexer 210 debug_info: boolean, if True include debugging info 211 """ 212 super().__init__(dexer, debug_info, ['-Xint']) 213 214 @property 215 def description(self): 216 return 'Art interpreter on host' 217 218 @property 219 def id(self): 220 return 'HInt' 221 222 def GetBisectionSearchArgs(self): 223 return None 224 225 226class TestRunnerArtOptOnHost(TestRunnerArtOnHost): 227 """Concrete test runner of optimizing compiler mode Art on host.""" 228 229 def __init__(self, dexer, debug_info): 230 """Constructor for the Art on host tester (optimizing). 231 232 Args: 233 dexer: string, defines dexer 234 debug_info: boolean, if True include debugging info 235 """ 236 super().__init__(dexer, debug_info, None) 237 238 @property 239 def description(self): 240 return 'Art optimizing on host' 241 242 @property 243 def id(self): 244 return 'HOpt' 245 246 def GetBisectionSearchArgs(self): 247 cmd_str = CommandListToCommandString( 248 self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:]) 249 return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 250 251 252class TestRunnerArtOnTarget(TestRunnerWithHostCompilation): 253 """Abstract test runner of Art on target.""" 254 255 def __init__(self, dexer, debug_info, device, extra_args=None): 256 """Constructor for the Art on target tester. 257 258 Args: 259 dexer: string, defines dexer 260 debug_info: boolean, if True include debugging info 261 device: string, target device serial number (or None) 262 extra_args: list of strings, extra arguments for dalvikvm 263 """ 264 super().__init__(dexer, debug_info) 265 self._test_env = DeviceTestEnv('jfuzz_', specific_device=device) 266 self._dalvik_cmd = ['dalvikvm'] 267 if extra_args is not None: 268 self._dalvik_cmd += extra_args 269 self._device = device 270 self._device_classpath = None 271 272 def CompileAndRunTest(self): 273 if self.CompileOnHost() == RetCode.SUCCESS: 274 self._device_classpath = self._test_env.PushClasspath('classes.dex') 275 cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'] 276 (output, retc) = self._test_env.RunCommand( 277 cmd, {'ANDROID_LOG_TAGS': '*:s'}) 278 with open(self.output_file, 'w') as run_out: 279 run_out.write(output) 280 else: 281 retc = RetCode.NOTCOMPILED 282 return retc 283 284 def GetBisectionSearchArgs(self): 285 cmd_str = CommandListToCommandString( 286 self._dalvik_cmd + ['-cp',self._device_classpath, 'Test']) 287 cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 288 if self._device: 289 cmd += ['--device-serial', self._device] 290 else: 291 cmd.append('--device') 292 return cmd 293 294 295class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget): 296 """Concrete test runner of interpreter mode Art on target.""" 297 298 def __init__(self, dexer, debug_info, device): 299 """Constructor for the Art on target tester (interpreter). 300 301 Args: 302 dexer: string, defines dexer 303 debug_info: boolean, if True include debugging info 304 device: string, target device serial number (or None) 305 """ 306 super().__init__(dexer, debug_info, device, ['-Xint']) 307 308 @property 309 def description(self): 310 return 'Art interpreter on target' 311 312 @property 313 def id(self): 314 return 'TInt' 315 316 def GetBisectionSearchArgs(self): 317 return None 318 319 320class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget): 321 """Concrete test runner of optimizing compiler mode Art on target.""" 322 323 def __init__(self, dexer, debug_info, device): 324 """Constructor for the Art on target tester (optimizing). 325 326 Args: 327 dexer: string, defines dexer 328 debug_info: boolean, if True include debugging info 329 device: string, target device serial number (or None) 330 """ 331 super().__init__(dexer, debug_info, device, None) 332 333 @property 334 def description(self): 335 return 'Art optimizing on target' 336 337 @property 338 def id(self): 339 return 'TOpt' 340 341 def GetBisectionSearchArgs(self): 342 cmd_str = CommandListToCommandString( 343 self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']) 344 cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)] 345 if self._device: 346 cmd += ['--device-serial', self._device] 347 else: 348 cmd.append('--device') 349 return cmd 350 351 352# 353# Tester class. 354# 355 356 357class JFuzzTester(object): 358 """Tester that runs JFuzz many times and report divergences.""" 359 360 def __init__(self, num_tests, device, mode1, mode2, jfuzz_args, 361 report_script, true_divergence_only, dexer, debug_info): 362 """Constructor for the tester. 363 364 Args: 365 num_tests: int, number of tests to run 366 device: string, target device serial number (or None) 367 mode1: string, execution mode for first runner 368 mode2: string, execution mode for second runner 369 jfuzz_args: list of strings, additional arguments for jfuzz 370 report_script: string, path to script called for each divergence 371 true_divergence_only: boolean, if True don't bisect timeout divergences 372 dexer: string, defines dexer 373 debug_info: boolean, if True include debugging info 374 """ 375 self._num_tests = num_tests 376 self._device = device 377 self._runner1 = GetExecutionModeRunner(dexer, debug_info, device, mode1) 378 self._runner2 = GetExecutionModeRunner(dexer, debug_info, device, mode2) 379 self._jfuzz_args = jfuzz_args 380 self._report_script = report_script 381 self._true_divergence_only = true_divergence_only 382 self._dexer = dexer 383 self._debug_info = debug_info 384 self._save_dir = None 385 self._results_dir = None 386 self._jfuzz_dir = None 387 # Statistics. 388 self._test = 0 389 self._num_success = 0 390 self._num_not_compiled = 0 391 self._num_not_run = 0 392 self._num_timed_out = 0 393 self._num_divergences = 0 394 395 def __enter__(self): 396 """On entry, enters new temp directory after saving current directory. 397 398 Raises: 399 FatalError: error when temp directory cannot be constructed 400 """ 401 self._save_dir = os.getcwd() 402 self._results_dir = mkdtemp(dir='/tmp/') 403 self._jfuzz_dir = mkdtemp(dir=self._results_dir) 404 if self._results_dir is None or self._jfuzz_dir is None: 405 raise FatalError('Cannot obtain temp directory') 406 os.chdir(self._jfuzz_dir) 407 return self 408 409 def __exit__(self, etype, evalue, etraceback): 410 """On exit, re-enters previously saved current directory and cleans up.""" 411 os.chdir(self._save_dir) 412 shutil.rmtree(self._jfuzz_dir) 413 if self._num_divergences == 0: 414 shutil.rmtree(self._results_dir) 415 416 def Run(self): 417 """Runs JFuzz many times and report divergences.""" 418 print() 419 print('**\n**** JFuzz Testing\n**') 420 print() 421 print('#Tests :', self._num_tests) 422 print('Device :', self._device) 423 print('Directory :', self._results_dir) 424 print('Exec-mode1:', self._runner1.description) 425 print('Exec-mode2:', self._runner2.description) 426 print('Dexer :', self._dexer) 427 print('Debug-info:', self._debug_info) 428 print() 429 self.ShowStats() 430 for self._test in range(1, self._num_tests + 1): 431 self.RunJFuzzTest() 432 self.ShowStats() 433 if self._num_divergences == 0: 434 print('\n\nsuccess (no divergences)\n') 435 else: 436 print('\n\nfailure (divergences)\n') 437 438 def ShowStats(self): 439 """Shows current statistics (on same line) while tester is running.""" 440 print('\rTests:', self._test, 441 'Success:', self._num_success, 442 'Not-compiled:', self._num_not_compiled, 443 'Not-run:', self._num_not_run, 444 'Timed-out:', self._num_timed_out, 445 'Divergences:', self._num_divergences, 446 end='') 447 sys.stdout.flush() 448 449 def RunJFuzzTest(self): 450 """Runs a single JFuzz test, comparing two execution modes.""" 451 self.ConstructTest() 452 retc1 = self._runner1.CompileAndRunTest() 453 retc2 = self._runner2.CompileAndRunTest() 454 self.CheckForDivergence(retc1, retc2) 455 self.CleanupTest() 456 457 def ConstructTest(self): 458 """Use JFuzz to generate next Test.java test. 459 460 Raises: 461 FatalError: error when jfuzz fails 462 """ 463 if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None) 464 != RetCode.SUCCESS): 465 raise FatalError('Unexpected error while running JFuzz') 466 467 def CheckForDivergence(self, retc1, retc2): 468 """Checks for divergences and updates statistics. 469 470 Args: 471 retc1: int, normalized return code of first runner 472 retc2: int, normalized return code of second runner 473 """ 474 if retc1 == retc2: 475 # No divergence in return code. 476 if retc1 == RetCode.SUCCESS: 477 # Both compilations and runs were successful, inspect generated output. 478 runner1_out = self._runner1.output_file 479 runner2_out = self._runner2.output_file 480 if not filecmp.cmp(runner1_out, runner2_out, shallow=False): 481 # Divergence in output. 482 self.ReportDivergence(retc1, retc2, is_output_divergence=True) 483 else: 484 # No divergence in output. 485 self._num_success += 1 486 elif retc1 == RetCode.TIMEOUT: 487 self._num_timed_out += 1 488 elif retc1 == RetCode.NOTCOMPILED: 489 self._num_not_compiled += 1 490 else: 491 self._num_not_run += 1 492 else: 493 # Divergence in return code. 494 if self._true_divergence_only: 495 # When only true divergences are requested, any divergence in return 496 # code where one is a time out is treated as a regular time out. 497 if RetCode.TIMEOUT in (retc1, retc2): 498 self._num_timed_out += 1 499 return 500 # When only true divergences are requested, a runtime crash in just 501 # the RI is treated as if not run at all. 502 if retc1 == RetCode.ERROR and retc2 == RetCode.SUCCESS: 503 if self._runner1.GetBisectionSearchArgs() is None: 504 self._num_not_run += 1 505 return 506 self.ReportDivergence(retc1, retc2, is_output_divergence=False) 507 508 def GetCurrentDivergenceDir(self): 509 return self._results_dir + '/divergence' + str(self._num_divergences) 510 511 def ReportDivergence(self, retc1, retc2, is_output_divergence): 512 """Reports and saves a divergence.""" 513 self._num_divergences += 1 514 print('\n' + str(self._num_divergences), end='') 515 if is_output_divergence: 516 print(' divergence in output') 517 else: 518 print(' divergence in return code: ' + retc1.name + ' vs. ' + 519 retc2.name) 520 # Save. 521 ddir = self.GetCurrentDivergenceDir() 522 os.mkdir(ddir) 523 for f in glob('*.txt') + ['Test.java']: 524 shutil.copy(f, ddir) 525 # Maybe run bisection bug search. 526 if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES: 527 self.MaybeBisectDivergence(retc1, retc2, is_output_divergence) 528 # Call reporting script. 529 if self._report_script: 530 self.RunReportScript(retc1, retc2, is_output_divergence) 531 532 def RunReportScript(self, retc1, retc2, is_output_divergence): 533 """Runs report script.""" 534 try: 535 title = "Divergence between {0} and {1} (found with fuzz testing)".format( 536 self._runner1.description, self._runner2.description) 537 # Prepare divergence comment. 538 jfuzz_cmd_and_version = subprocess.check_output( 539 ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True) 540 (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(') 541 # Strip right parenthesis and new line. 542 jfuzz_ver = jfuzz_ver[:-2] 543 jfuzz_args = ['\'-{0}\''.format(arg) 544 for arg in jfuzz_cmd_str.strip().split(' -')][1:] 545 wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args] 546 repro_cmd_str = (os.path.basename(__file__) + 547 ' --num_tests=1 --dexer=' + self._dexer + 548 (' --debug_info ' if self._debug_info else ' ') + 549 ' '.join(wrapped_args)) 550 comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format( 551 jfuzz_ver, jfuzz_cmd_str, repro_cmd_str) 552 if is_output_divergence: 553 (output, _, _) = RunCommandForOutput( 554 ['diff', self._runner1.output_file, self._runner2.output_file], 555 None, subprocess.PIPE, subprocess.STDOUT) 556 comment += 'Diff:\n' + output 557 else: 558 comment += '{0} vs {1}\n'.format(retc1, retc2) 559 # Prepare report script command. 560 script_cmd = [self._report_script, title, comment] 561 ddir = self.GetCurrentDivergenceDir() 562 bisection_out_files = glob(ddir + '/*_bisection_out.txt') 563 if bisection_out_files: 564 script_cmd += ['--bisection_out', bisection_out_files[0]] 565 subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL) 566 except subprocess.CalledProcessError as err: 567 print('Failed to run report script.\n', err) 568 569 def RunBisectionSearch(self, args, expected_retcode, expected_output, 570 runner_id): 571 ddir = self.GetCurrentDivergenceDir() 572 outfile_path = ddir + '/' + runner_id + '_bisection_out.txt' 573 logfile_path = ddir + '/' + runner_id + '_bisection_log.txt' 574 errfile_path = ddir + '/' + runner_id + '_bisection_err.txt' 575 args = list(args) + ['--logfile', logfile_path, '--cleanup'] 576 args += ['--expected-retcode', expected_retcode.name] 577 if expected_output: 578 args += ['--expected-output', expected_output] 579 bisection_search_path = os.path.join( 580 GetEnvVariableOrError('ANDROID_BUILD_TOP'), 581 'art/tools/bisection_search/bisection_search.py') 582 if RunCommand([bisection_search_path] + args, out=outfile_path, 583 err=errfile_path, timeout=300) == RetCode.TIMEOUT: 584 print('Bisection search TIMEOUT') 585 586 def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence): 587 bisection_args1 = self._runner1.GetBisectionSearchArgs() 588 bisection_args2 = self._runner2.GetBisectionSearchArgs() 589 if is_output_divergence: 590 maybe_output1 = self._runner1.output_file 591 maybe_output2 = self._runner2.output_file 592 else: 593 maybe_output1 = maybe_output2 = None 594 if bisection_args1 is not None: 595 self.RunBisectionSearch(bisection_args1, retc2, maybe_output2, 596 self._runner1.id) 597 if bisection_args2 is not None: 598 self.RunBisectionSearch(bisection_args2, retc1, maybe_output1, 599 self._runner2.id) 600 601 def CleanupTest(self): 602 """Cleans up after a single test run.""" 603 for file_name in os.listdir(self._jfuzz_dir): 604 file_path = os.path.join(self._jfuzz_dir, file_name) 605 if os.path.isfile(file_path): 606 os.unlink(file_path) 607 elif os.path.isdir(file_path): 608 shutil.rmtree(file_path) 609 610 611def main(): 612 # Handle arguments. 613 parser = argparse.ArgumentParser() 614 parser.add_argument('--num_tests', default=10000, type=int, 615 help='number of tests to run') 616 parser.add_argument('--device', help='target device serial number') 617 parser.add_argument('--mode1', default='ri', 618 help='execution mode 1 (default: ri)') 619 parser.add_argument('--mode2', default='hopt', 620 help='execution mode 2 (default: hopt)') 621 parser.add_argument('--report_script', 622 help='script called for each divergence') 623 parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args', 624 action='append', 625 help='argument for jfuzz') 626 parser.add_argument('--true_divergence', default=False, action='store_true', 627 help='do not bisect timeout divergences') 628 parser.add_argument('--dexer', default='dx', type=str, 629 help='defines dexer as dx or d8 (default: dx)') 630 parser.add_argument('--debug_info', default=False, action='store_true', 631 help='include debugging info') 632 args = parser.parse_args() 633 if args.mode1 == args.mode2: 634 raise FatalError('Identical execution modes given') 635 # Run the JFuzz tester. 636 with JFuzzTester(args.num_tests, 637 args.device, 638 args.mode1, args.mode2, 639 args.jfuzz_args, 640 args.report_script, 641 args.true_divergence, 642 args.dexer, 643 args.debug_info) as fuzzer: 644 fuzzer.Run() 645 646if __name__ == '__main__': 647 main() 648