• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Tests the functionality of the fuzz_target module."""
15
16import os
17import tempfile
18import unittest
19from unittest import mock
20
21import certifi
22# Importing this later causes import failures with pytest for some reason.
23# TODO(ochang): Figure out why.
24import parameterized
25import google.cloud.ndb  # pylint: disable=unused-import
26from pyfakefs import fake_filesystem_unittest
27from clusterfuzz.fuzz import engine
28
29import clusterfuzz_deployment
30import fuzz_target
31import test_helpers
32import workspace_utils
33
34# NOTE: This integration test relies on
35# https://github.com/google/oss-fuzz/tree/master/projects/example project.
36EXAMPLE_PROJECT = 'example'
37
38# An example fuzzer that triggers an error.
39EXAMPLE_FUZZER = 'example_crash_fuzzer'
40
41# Mock return values for engine_impl.reproduce.
42EXECUTE_SUCCESS_RESULT = engine.ReproduceResult([], 0, 0, '')
43EXECUTE_FAILURE_RESULT = engine.ReproduceResult([], 1, 0, '')
44
45
46def _create_config(**kwargs):
47  """Creates a config object and then sets every attribute that is a key in
48  |kwargs| to the corresponding value. Asserts that each key in |kwargs| is an
49  attribute of Config."""
50  defaults = {
51      'is_github': True,
52      'oss_fuzz_project_name': EXAMPLE_PROJECT,
53      'workspace': '/workspace'
54  }
55  for default_key, default_value in defaults.items():
56    if default_key not in kwargs:
57      kwargs[default_key] = default_value
58
59  return test_helpers.create_run_config(**kwargs)
60
61
62def _create_deployment(**kwargs):
63  config = _create_config(**kwargs)
64  workspace = workspace_utils.Workspace(config)
65  return clusterfuzz_deployment.get_clusterfuzz_deployment(config, workspace)
66
67
68@mock.patch('utils.get_container_name', return_value='container')
69class IsReproducibleTest(fake_filesystem_unittest.TestCase):
70  """Tests the is_reproducible method in the fuzz_target.FuzzTarget class."""
71
72  def setUp(self):
73    """Sets up example fuzz target to test is_reproducible method."""
74    self.fuzz_target_name = 'fuzz-target'
75    deployment = _create_deployment()
76    self.config = deployment.config
77    self.workspace = deployment.workspace
78    self.fuzz_target_path = os.path.join(self.workspace.out,
79                                         self.fuzz_target_name)
80    self.setUpPyfakefs()
81    self.fs.create_file(self.fuzz_target_path)
82    self.testcase_path = '/testcase'
83    self.fs.create_file(self.testcase_path)
84
85    self.target = fuzz_target.FuzzTarget(self.fuzz_target_path,
86                                         fuzz_target.REPRODUCE_ATTEMPTS,
87                                         self.workspace, deployment,
88                                         deployment.config)
89
90    # ClusterFuzz requires ROOT_DIR.
91    root_dir = os.environ['ROOT_DIR']
92    test_helpers.patch_environ(self, empty=True)
93    os.environ['ROOT_DIR'] = root_dir
94
95  def test_reproducible(self, _):
96    """Tests that is_reproducible returns True if crash is detected and that
97    is_reproducible uses the correct command to reproduce a crash."""
98    all_repro = [EXECUTE_FAILURE_RESULT] * fuzz_target.REPRODUCE_ATTEMPTS
99    with mock.patch('clusterfuzz.fuzz.get_engine') as mock_get_engine:
100      mock_get_engine().reproduce.side_effect = all_repro
101
102      result = self.target.is_reproducible(self.testcase_path,
103                                           self.fuzz_target_path)
104      mock_get_engine().reproduce.assert_called_once_with(
105          '/workspace/build-out/fuzz-target',
106          '/testcase',
107          arguments=[],
108          max_time=30)
109      self.assertTrue(result)
110      self.assertEqual(1, mock_get_engine().reproduce.call_count)
111
112  def test_flaky(self, _):
113    """Tests that is_reproducible returns True if crash is detected on the last
114    attempt."""
115    last_time_repro = [EXECUTE_SUCCESS_RESULT] * 9 + [EXECUTE_FAILURE_RESULT]
116    with mock.patch('clusterfuzz.fuzz.get_engine') as mock_get_engine:
117      mock_get_engine().reproduce.side_effect = last_time_repro
118      self.assertTrue(
119          self.target.is_reproducible(self.testcase_path,
120                                      self.fuzz_target_path))
121      self.assertEqual(fuzz_target.REPRODUCE_ATTEMPTS,
122                       mock_get_engine().reproduce.call_count)
123
124  def test_nonexistent_fuzzer(self, _):
125    """Tests that is_reproducible raises an error if it could not attempt
126    reproduction because the fuzzer doesn't exist."""
127    with self.assertRaises(fuzz_target.ReproduceError):
128      self.target.is_reproducible(self.testcase_path, '/non-existent-path')
129
130  def test_unreproducible(self, _):
131    """Tests that is_reproducible returns False for a crash that did not
132    reproduce."""
133    all_unrepro = [EXECUTE_SUCCESS_RESULT] * fuzz_target.REPRODUCE_ATTEMPTS
134    with mock.patch('clusterfuzz.fuzz.get_engine') as mock_get_engine:
135      mock_get_engine().reproduce.side_effect = all_unrepro
136      result = self.target.is_reproducible(self.testcase_path,
137                                           self.fuzz_target_path)
138      self.assertFalse(result)
139
140
141class IsCrashReportableTest(fake_filesystem_unittest.TestCase):
142  """Tests the is_crash_reportable method of FuzzTarget."""
143
144  def setUp(self):
145    """Sets up example fuzz target to test is_crash_reportable method."""
146    self.setUpPyfakefs()
147    self.fuzz_target_path = '/example/do_stuff_fuzzer'
148    deployment = _create_deployment()
149    self.target = fuzz_target.FuzzTarget(self.fuzz_target_path, 100,
150                                         deployment.workspace, deployment,
151                                         deployment.config)
152    self.oss_fuzz_build_path = '/oss-fuzz-build'
153    self.fs.create_file(self.fuzz_target_path)
154    self.oss_fuzz_target_path = os.path.join(
155        self.oss_fuzz_build_path, os.path.basename(self.fuzz_target_path))
156    self.fs.create_file(self.oss_fuzz_target_path)
157    self.testcase_path = '/testcase'
158    self.fs.create_file(self.testcase_path, contents='')
159
160    # Do this to prevent pyfakefs from messing with requests.
161    self.fs.add_real_directory(os.path.dirname(certifi.__file__))
162
163  @mock.patch('fuzz_target.FuzzTarget.is_reproducible',
164              side_effect=[True, False])
165  @mock.patch('logging.info')
166  def test_new_reproducible_crash(self, mock_info, _):
167    """Tests that a new reproducible crash returns True."""
168    with tempfile.TemporaryDirectory() as tmp_dir:
169      self.target.out_dir = tmp_dir
170      self.assertTrue(self.target.is_crash_reportable(self.testcase_path))
171    mock_info.assert_called_with(
172        'The crash is not reproducible on previous build. '
173        'Code change (pr/commit) introduced crash.')
174
175  # yapf: disable
176  @parameterized.parameterized.expand([
177      # Reproducible on PR build, but also reproducible on OSS-Fuzz.
178      ([True, True],),
179
180      # Not reproducible on PR build, but somehow reproducible on OSS-Fuzz.
181      # Unlikely to happen in real world except if test is flaky.
182      ([False, True],),
183
184      # Not reproducible on PR build, and not reproducible on OSS-Fuzz.
185      ([False, False],),
186  ])
187  # yapf: enable
188  def test_invalid_crash(self, is_reproducible_retvals):
189    """Tests that a nonreportable crash causes the method to return False."""
190    with mock.patch('fuzz_target.FuzzTarget.is_reproducible',
191                    side_effect=is_reproducible_retvals):
192      with mock.patch('clusterfuzz_deployment.OSSFuzz.download_latest_build',
193                      return_value=self.oss_fuzz_build_path):
194        self.assertFalse(self.target.is_crash_reportable(self.testcase_path))
195
196  @mock.patch('logging.info')
197  @mock.patch('fuzz_target.FuzzTarget.is_reproducible', return_value=[True])
198  def test_reproducible_no_oss_fuzz_target(self, _, mock_info):
199    """Tests that is_crash_reportable returns True when a crash reproduces on
200    the PR build but the target is not in the OSS-Fuzz build (usually because it
201    is new)."""
202    os.remove(self.oss_fuzz_target_path)
203
204    def is_reproducible_side_effect(_, target_path):
205      if os.path.dirname(target_path) == self.oss_fuzz_build_path:
206        raise fuzz_target.ReproduceError()
207      return True
208
209    with mock.patch(
210        'fuzz_target.FuzzTarget.is_reproducible',
211        side_effect=is_reproducible_side_effect) as mock_is_reproducible:
212      with mock.patch('clusterfuzz_deployment.OSSFuzz.download_latest_build',
213                      return_value=self.oss_fuzz_build_path):
214        self.assertTrue(self.target.is_crash_reportable(self.testcase_path))
215    mock_is_reproducible.assert_any_call(self.testcase_path,
216                                         self.oss_fuzz_target_path)
217    mock_info.assert_called_with(
218        'Could not run previous build of target to determine if this code '
219        'change (pr/commit) introduced crash. Assuming crash was newly '
220        'introduced.')
221
222
223if __name__ == '__main__':
224  unittest.main()
225