1# Copyright 2021 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Tests for running fuzzers.""" 15import json 16import os 17import sys 18import shutil 19import tempfile 20import unittest 21from unittest import mock 22 23import parameterized 24from pyfakefs import fake_filesystem_unittest 25 26import build_fuzzers 27import fuzz_target 28import run_fuzzers 29 30# pylint: disable=wrong-import-position 31INFRA_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 32sys.path.append(INFRA_DIR) 33 34import helper 35import test_helpers 36 37# NOTE: This integration test relies on 38# https://github.com/google/oss-fuzz/tree/master/projects/example project. 39EXAMPLE_PROJECT = 'example' 40 41# Location of files used for testing. 42TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 43 'test_data') 44 45MEMORY_FUZZER_DIR = os.path.join(TEST_DATA_PATH, 'memory') 46MEMORY_FUZZER = 'curl_fuzzer_memory' 47 48UNDEFINED_FUZZER_DIR = os.path.join(TEST_DATA_PATH, 'undefined') 49UNDEFINED_FUZZER = 'curl_fuzzer_undefined' 50 51FUZZ_SECONDS = 10 52 53 54class RunFuzzerIntegrationTestMixin: # pylint: disable=too-few-public-methods,invalid-name 55 """Mixin for integration test classes that runbuild_fuzzers on builds of a 56 specific sanitizer.""" 57 # These must be defined by children. 58 FUZZER_DIR = None 59 FUZZER = None 60 61 def setUp(self): 62 """Patch the environ so that we can execute runner scripts.""" 63 test_helpers.patch_environ(self, runner=True) 64 65 def _test_run_with_sanitizer(self, fuzzer_dir, sanitizer): 66 """Calls run_fuzzers on fuzzer_dir and |sanitizer| and asserts 67 the run succeeded and that no bug was found.""" 68 with test_helpers.temp_dir_copy(fuzzer_dir) as fuzzer_dir_copy: 69 config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS, 70 workspace=fuzzer_dir_copy, 71 oss_fuzz_project_name='curl', 72 sanitizer=sanitizer) 73 result = run_fuzzers.run_fuzzers(config) 74 self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND) 75 76 77@unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 78 'INTEGRATION_TESTS=1 not set') 79class RunMemoryFuzzerIntegrationTest(RunFuzzerIntegrationTestMixin, 80 unittest.TestCase): 81 """Integration test for build_fuzzers with an MSAN build.""" 82 FUZZER_DIR = MEMORY_FUZZER_DIR 83 FUZZER = MEMORY_FUZZER 84 85 def test_run_with_memory_sanitizer(self): 86 """Tests run_fuzzers with a valid MSAN build.""" 87 self._test_run_with_sanitizer(self.FUZZER_DIR, 'memory') 88 89 90@unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 91 'INTEGRATION_TESTS=1 not set') 92class RunUndefinedFuzzerIntegrationTest(RunFuzzerIntegrationTestMixin, 93 unittest.TestCase): 94 """Integration test for build_fuzzers with an UBSAN build.""" 95 FUZZER_DIR = UNDEFINED_FUZZER_DIR 96 FUZZER = UNDEFINED_FUZZER 97 98 def test_run_with_undefined_sanitizer(self): 99 """Tests run_fuzzers with a valid UBSAN build.""" 100 self._test_run_with_sanitizer(self.FUZZER_DIR, 'undefined') 101 102 103class BaseFuzzTargetRunnerTest(unittest.TestCase): 104 """Tests BaseFuzzTargetRunner.""" 105 106 def _create_runner(self, **kwargs): # pylint: disable=no-self-use 107 defaults = { 108 'fuzz_seconds': FUZZ_SECONDS, 109 'oss_fuzz_project_name': EXAMPLE_PROJECT 110 } 111 for default_key, default_value in defaults.items(): 112 if default_key not in kwargs: 113 kwargs[default_key] = default_value 114 115 config = test_helpers.create_run_config(**kwargs) 116 return run_fuzzers.BaseFuzzTargetRunner(config) 117 118 def _test_initialize_fail(self, expected_error_args, **create_runner_kwargs): 119 with mock.patch('logging.error') as mock_error: 120 runner = self._create_runner(**create_runner_kwargs) 121 self.assertFalse(runner.initialize()) 122 mock_error.assert_called_with(*expected_error_args) 123 124 @parameterized.parameterized.expand([(0,), (None,), (-1,)]) 125 def test_initialize_invalid_fuzz_seconds(self, fuzz_seconds): 126 """Tests initialize fails with an invalid fuzz seconds.""" 127 expected_error_args = ('Fuzz_seconds argument must be greater than 1, ' 128 'but was: %s.', fuzz_seconds) 129 with tempfile.TemporaryDirectory() as tmp_dir: 130 out_path = os.path.join(tmp_dir, 'build-out') 131 os.mkdir(out_path) 132 with mock.patch('utils.get_fuzz_targets') as mock_get_fuzz_targets: 133 mock_get_fuzz_targets.return_value = [ 134 os.path.join(out_path, 'fuzz_target') 135 ] 136 self._test_initialize_fail(expected_error_args, 137 fuzz_seconds=fuzz_seconds, 138 workspace=tmp_dir) 139 140 def test_initialize_no_out_dir(self): 141 """Tests initialize fails with no out dir.""" 142 with tempfile.TemporaryDirectory() as tmp_dir: 143 out_path = os.path.join(tmp_dir, 'build-out') 144 expected_error_args = ('Out directory: %s does not exist.', out_path) 145 self._test_initialize_fail(expected_error_args, workspace=tmp_dir) 146 147 def test_initialize_nonempty_artifacts(self): 148 """Tests initialize with a file artifacts path.""" 149 with tempfile.TemporaryDirectory() as tmp_dir: 150 out_path = os.path.join(tmp_dir, 'build-out') 151 os.mkdir(out_path) 152 os.makedirs(os.path.join(tmp_dir, 'out')) 153 artifacts_path = os.path.join(tmp_dir, 'out', 'artifacts') 154 with open(artifacts_path, 'w') as artifacts_handle: 155 artifacts_handle.write('fake') 156 expected_error_args = ( 157 'Artifacts path: %s exists and is not an empty directory.', 158 artifacts_path) 159 self._test_initialize_fail(expected_error_args, workspace=tmp_dir) 160 161 def test_initialize_bad_artifacts(self): 162 """Tests initialize with a non-empty artifacts path.""" 163 with tempfile.TemporaryDirectory() as tmp_dir: 164 out_path = os.path.join(tmp_dir, 'build-out') 165 os.mkdir(out_path) 166 artifacts_path = os.path.join(tmp_dir, 'out', 'artifacts') 167 os.makedirs(artifacts_path) 168 artifact_path = os.path.join(artifacts_path, 'artifact') 169 with open(artifact_path, 'w') as artifact_handle: 170 artifact_handle.write('fake') 171 expected_error_args = ( 172 'Artifacts path: %s exists and is not an empty directory.', 173 artifacts_path) 174 self._test_initialize_fail(expected_error_args, workspace=tmp_dir) 175 176 @mock.patch('utils.get_fuzz_targets') 177 @mock.patch('logging.error') 178 def test_initialize_empty_artifacts(self, mock_log_error, 179 mock_get_fuzz_targets): 180 """Tests initialize with an empty artifacts dir.""" 181 mock_get_fuzz_targets.return_value = ['fuzz-target'] 182 with tempfile.TemporaryDirectory() as tmp_dir: 183 out_path = os.path.join(tmp_dir, 'build-out') 184 os.mkdir(out_path) 185 artifacts_path = os.path.join(tmp_dir, 'out', 'artifacts') 186 os.makedirs(artifacts_path) 187 runner = self._create_runner(workspace=tmp_dir) 188 self.assertTrue(runner.initialize()) 189 mock_log_error.assert_not_called() 190 self.assertTrue(os.path.isdir(artifacts_path)) 191 192 @mock.patch('utils.get_fuzz_targets') 193 @mock.patch('logging.error') 194 def test_initialize_no_artifacts(self, mock_log_error, mock_get_fuzz_targets): 195 """Tests initialize with no artifacts dir (the expected setting).""" 196 mock_get_fuzz_targets.return_value = ['fuzz-target'] 197 with tempfile.TemporaryDirectory() as tmp_dir: 198 out_path = os.path.join(tmp_dir, 'build-out') 199 os.mkdir(out_path) 200 runner = self._create_runner(workspace=tmp_dir) 201 self.assertTrue(runner.initialize()) 202 mock_log_error.assert_not_called() 203 self.assertTrue(os.path.isdir(os.path.join(tmp_dir, 'out', 'artifacts'))) 204 205 def test_initialize_no_fuzz_targets(self): 206 """Tests initialize with no fuzz targets.""" 207 with tempfile.TemporaryDirectory() as tmp_dir: 208 out_path = os.path.join(tmp_dir, 'build-out') 209 os.makedirs(out_path) 210 expected_error_args = ('No fuzz targets were found in out directory: %s.', 211 out_path) 212 self._test_initialize_fail(expected_error_args, workspace=tmp_dir) 213 214 def test_get_fuzz_target_artifact(self): 215 """Tests that get_fuzz_target_artifact works as intended.""" 216 with tempfile.TemporaryDirectory() as tmp_dir: 217 runner = self._create_runner(workspace=tmp_dir) 218 crashes_dir = 'crashes-dir' 219 runner.crashes_dir = crashes_dir 220 artifact_name = 'artifact-name' 221 target = mock.MagicMock() 222 target_name = 'target_name' 223 target.target_name = target_name 224 225 fuzz_target_artifact = runner.get_fuzz_target_artifact( 226 target, artifact_name) 227 expected_fuzz_target_artifact = os.path.join( 228 tmp_dir, 'out', 'artifacts', 'target_name-address-artifact-name') 229 230 self.assertEqual(fuzz_target_artifact, expected_fuzz_target_artifact) 231 232 233class CiFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): 234 """Tests that CiFuzzTargetRunner works as intended.""" 235 236 def setUp(self): 237 self.setUpPyfakefs() 238 239 @mock.patch('utils.get_fuzz_targets') 240 @mock.patch('run_fuzzers.CiFuzzTargetRunner.run_fuzz_target') 241 @mock.patch('run_fuzzers.CiFuzzTargetRunner.create_fuzz_target_obj') 242 def test_run_fuzz_targets_quits(self, mock_create_fuzz_target_obj, 243 mock_run_fuzz_target, mock_get_fuzz_targets): 244 """Tests that run_fuzz_targets quits on the first crash it finds.""" 245 workspace = 'workspace' 246 out_path = os.path.join(workspace, 'build-out') 247 self.fs.create_dir(out_path) 248 config = test_helpers.create_run_config( 249 fuzz_seconds=FUZZ_SECONDS, 250 workspace=workspace, 251 oss_fuzz_project_name=EXAMPLE_PROJECT) 252 runner = run_fuzzers.CiFuzzTargetRunner(config) 253 254 mock_get_fuzz_targets.return_value = ['target1', 'target2'] 255 runner.initialize() 256 testcase = os.path.join(workspace, 'testcase') 257 self.fs.create_file(testcase) 258 stacktrace = 'stacktrace' 259 corpus_dir = 'corpus' 260 self.fs.create_dir(corpus_dir) 261 mock_run_fuzz_target.return_value = fuzz_target.FuzzResult( 262 testcase, stacktrace, corpus_dir) 263 magic_mock = mock.MagicMock() 264 magic_mock.target_name = 'target1' 265 mock_create_fuzz_target_obj.return_value = magic_mock 266 self.assertTrue(runner.run_fuzz_targets()) 267 self.assertIn('target1-address-testcase', 268 os.listdir(runner.workspace.artifacts)) 269 self.assertEqual(mock_run_fuzz_target.call_count, 1) 270 271 272class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): 273 """Tests that BatchFuzzTargetRunnerTest works as intended.""" 274 WORKSPACE = 'workspace' 275 STACKTRACE = 'stacktrace' 276 CORPUS_DIR = 'corpus' 277 278 def setUp(self): 279 self.setUpPyfakefs() 280 out_dir = os.path.join(self.WORKSPACE, 'build-out') 281 self.fs.create_dir(out_dir) 282 self.testcase1 = os.path.join(out_dir, 'testcase-aaa') 283 self.fs.create_file(self.testcase1) 284 self.testcase2 = os.path.join(out_dir, 'testcase-bbb') 285 self.fs.create_file(self.testcase2) 286 self.config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS, 287 workspace=self.WORKSPACE, 288 is_github=True) 289 290 @mock.patch('utils.get_fuzz_targets', return_value=['target1', 'target2']) 291 @mock.patch('clusterfuzz_deployment.ClusterFuzzLite.upload_build', 292 return_value=True) 293 @mock.patch('run_fuzzers.BatchFuzzTargetRunner.run_fuzz_target') 294 @mock.patch('run_fuzzers.BatchFuzzTargetRunner.create_fuzz_target_obj') 295 def test_run_fuzz_targets_quits(self, mock_create_fuzz_target_obj, 296 mock_run_fuzz_target, _, __): 297 """Tests that run_fuzz_targets doesn't quit on the first crash it finds.""" 298 runner = run_fuzzers.BatchFuzzTargetRunner(self.config) 299 runner.initialize() 300 301 call_count = 0 302 303 def mock_run_fuzz_target_impl(_): 304 nonlocal call_count 305 if call_count == 0: 306 testcase = self.testcase1 307 elif call_count == 1: 308 testcase = self.testcase2 309 assert call_count != 2 310 call_count += 1 311 if not os.path.exists(self.CORPUS_DIR): 312 self.fs.create_dir(self.CORPUS_DIR) 313 return fuzz_target.FuzzResult(testcase, self.STACKTRACE, self.CORPUS_DIR) 314 315 mock_run_fuzz_target.side_effect = mock_run_fuzz_target_impl 316 magic_mock = mock.MagicMock() 317 magic_mock.target_name = 'target1' 318 mock_create_fuzz_target_obj.return_value = magic_mock 319 self.assertTrue(runner.run_fuzz_targets()) 320 self.assertEqual(mock_run_fuzz_target.call_count, 2) 321 322 @mock.patch('run_fuzzers.BaseFuzzTargetRunner.run_fuzz_targets', 323 return_value=False) 324 @mock.patch('clusterfuzz_deployment.ClusterFuzzLite.upload_crashes') 325 def test_run_fuzz_targets_upload_crashes_and_builds(self, mock_upload_crashes, 326 _): 327 """Tests that run_fuzz_targets uploads crashes and builds correctly.""" 328 runner = run_fuzzers.BatchFuzzTargetRunner(self.config) 329 # TODO(metzman): Don't rely on this failing gracefully. 330 runner.initialize() 331 332 self.assertFalse(runner.run_fuzz_targets()) 333 self.assertEqual(mock_upload_crashes.call_count, 1) 334 335 336@unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 337 'INTEGRATION_TESTS=1 not set') 338class CoverageReportIntegrationTest(unittest.TestCase): 339 """Integration tests for coverage reports.""" 340 SANITIZER = 'coverage' 341 342 def setUp(self): 343 test_helpers.patch_environ(self, runner=True) 344 345 @mock.patch('filestore.github_actions._upload_artifact_with_upload_js') 346 def test_coverage_report(self, _): 347 """Tests generation of coverage reports end-to-end, from building to 348 generation.""" 349 350 with test_helpers.docker_temp_dir() as temp_dir: 351 shared = os.path.join(temp_dir, 'shared') 352 os.mkdir(shared) 353 copy_command = ('cp -r /opt/code_coverage /shared && ' 354 'cp $(which llvm-profdata) /shared && ' 355 'cp $(which llvm-cov) /shared') 356 assert helper.docker_run([ 357 '-v', f'{shared}:/shared', 'gcr.io/oss-fuzz-base/base-runner', 'bash', 358 '-c', copy_command 359 ]) 360 361 os.environ['CODE_COVERAGE_SRC'] = os.path.join(shared, 'code_coverage') 362 os.environ['PATH'] += os.pathsep + shared 363 # Do coverage build. 364 build_config = test_helpers.create_build_config( 365 oss_fuzz_project_name=EXAMPLE_PROJECT, 366 project_repo_name='oss-fuzz', 367 workspace=temp_dir, 368 commit_sha='0b95fe1039ed7c38fea1f97078316bfc1030c523', 369 base_commit='da0746452433dc18bae699e355a9821285d863c8', 370 sanitizer=self.SANITIZER, 371 is_github=True, 372 # Needed for test not to fail because of permissions issues. 373 bad_build_check=False) 374 self.assertTrue(build_fuzzers.build_fuzzers(build_config)) 375 376 # TODO(metzman): Get rid of this here and make 'compile' do this. 377 chmod_command = ('chmod -R +r /out && ' 378 'find /out -type d -exec chmod +x {} +') 379 380 assert helper.docker_run([ 381 '-v', f'{os.path.join(temp_dir, "build-out")}:/out', 382 'gcr.io/oss-fuzz-base/base-builder', 'bash', '-c', chmod_command 383 ]) 384 385 # Generate report. 386 run_config = test_helpers.create_run_config(fuzz_seconds=FUZZ_SECONDS, 387 workspace=temp_dir, 388 sanitizer=self.SANITIZER, 389 run_fuzzers_mode='coverage', 390 is_github=True) 391 result = run_fuzzers.run_fuzzers(run_config) 392 self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND) 393 expected_summary_path = os.path.join( 394 TEST_DATA_PATH, 'example_coverage_report_summary.json') 395 with open(expected_summary_path) as file_handle: 396 expected_summary = json.loads(file_handle.read()) 397 actual_summary_path = os.path.join(temp_dir, 'cifuzz-coverage', 398 'report', 'linux', 'summary.json') 399 with open(actual_summary_path) as file_handle: 400 actual_summary = json.loads(file_handle.read()) 401 self.assertEqual(expected_summary, actual_summary) 402 403 404@unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 405 'INTEGRATION_TESTS=1 not set') 406class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin, 407 unittest.TestCase): 408 """Integration tests for build_fuzzers with an ASAN build.""" 409 410 BUILD_DIR_NAME = 'cifuzz-latest-build' 411 412 def test_new_bug_found(self): 413 """Tests run_fuzzers with a valid ASAN build.""" 414 # Set the first return value to True, then the second to False to 415 # emulate a bug existing in the current PR but not on the downloaded 416 # OSS-Fuzz build. 417 with mock.patch('fuzz_target.FuzzTarget.is_reproducible', 418 side_effect=[True, False]): 419 with tempfile.TemporaryDirectory() as tmp_dir: 420 workspace = os.path.join(tmp_dir, 'workspace') 421 shutil.copytree(TEST_DATA_PATH, workspace) 422 config = test_helpers.create_run_config( 423 fuzz_seconds=FUZZ_SECONDS, 424 workspace=workspace, 425 oss_fuzz_project_name=EXAMPLE_PROJECT) 426 result = run_fuzzers.run_fuzzers(config) 427 self.assertEqual(result, run_fuzzers.RunFuzzersResult.BUG_FOUND) 428 429 @mock.patch('fuzz_target.FuzzTarget.is_reproducible', 430 side_effect=[True, True]) 431 def test_old_bug_found(self, _): 432 """Tests run_fuzzers with a bug found in OSS-Fuzz before.""" 433 with tempfile.TemporaryDirectory() as tmp_dir: 434 workspace = os.path.join(tmp_dir, 'workspace') 435 shutil.copytree(TEST_DATA_PATH, workspace) 436 config = test_helpers.create_run_config( 437 fuzz_seconds=FUZZ_SECONDS, 438 workspace=workspace, 439 oss_fuzz_project_name=EXAMPLE_PROJECT) 440 result = run_fuzzers.run_fuzzers(config) 441 self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND) 442 443 def test_invalid_build(self): 444 """Tests run_fuzzers with an invalid ASAN build.""" 445 with tempfile.TemporaryDirectory() as tmp_dir: 446 out_path = os.path.join(tmp_dir, 'build-out') 447 os.mkdir(out_path) 448 config = test_helpers.create_run_config( 449 fuzz_seconds=FUZZ_SECONDS, 450 workspace=tmp_dir, 451 oss_fuzz_project_name=EXAMPLE_PROJECT) 452 result = run_fuzzers.run_fuzzers(config) 453 self.assertEqual(result, run_fuzzers.RunFuzzersResult.ERROR) 454 455 456class GetFuzzTargetRunnerTest(unittest.TestCase): 457 """Tests for get_fuzz_fuzz_target_runner.""" 458 459 @parameterized.parameterized.expand([ 460 ('batch', run_fuzzers.BatchFuzzTargetRunner), 461 ('ci', run_fuzzers.CiFuzzTargetRunner), 462 ('coverage', run_fuzzers.CoverageTargetRunner) 463 ]) 464 def test_get_fuzz_target_runner(self, run_fuzzers_mode, 465 fuzz_target_runner_cls): 466 """Tests that get_fuzz_target_runner returns the correct runner based on the 467 specified run_fuzzers_mode.""" 468 with tempfile.TemporaryDirectory() as tmp_dir: 469 run_config = test_helpers.create_run_config( 470 fuzz_seconds=FUZZ_SECONDS, 471 workspace=tmp_dir, 472 oss_fuzz_project_name='example', 473 run_fuzzers_mode=run_fuzzers_mode) 474 runner = run_fuzzers.get_fuzz_target_runner(run_config) 475 self.assertTrue(isinstance(runner, fuzz_target_runner_cls)) 476 477 478if __name__ == '__main__': 479 unittest.main() 480