• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2021 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Tests for get_coverage.py"""
15import os
16import json
17import unittest
18from unittest import mock
19
20from pyfakefs import fake_filesystem_unittest
21import pytest
22
23import get_coverage
24
25# pylint: disable=protected-access
26
27TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
28                              'test_data')
29
30PROJECT_NAME = 'curl'
31REPO_PATH = '/src/curl'
32FUZZ_TARGET = 'curl_fuzzer'
33PROJECT_COV_JSON_FILENAME = 'example_curl_cov.json'
34FUZZ_TARGET_COV_JSON_FILENAME = 'example_curl_fuzzer_cov.json'
35INVALID_TARGET = 'not-a-fuzz-target'
36
37with open(os.path.join(TEST_DATA_PATH,
38                       PROJECT_COV_JSON_FILENAME),) as cov_file_handle:
39  PROJECT_COV_INFO = json.loads(cov_file_handle.read())
40
41
42class GetOssFuzzFuzzerStatsDirUrlTest(unittest.TestCase):
43  """Tests _get_oss_fuzz_fuzzer_stats_dir_url."""
44
45  @mock.patch('http_utils.get_json_from_url',
46              return_value={
47                  'fuzzer_stats_dir':
48                      'gs://oss-fuzz-coverage/systemd/fuzzer_stats/20210303'
49              })
50  def test_get_valid_project(self, mock_get_json_from_url):
51    """Tests that a project's coverage report can be downloaded and parsed.
52
53    NOTE: This test relies on the PROJECT_NAME repo's coverage report.
54    The "example" project was not used because it has no coverage reports.
55    """
56    result = get_coverage._get_oss_fuzz_fuzzer_stats_dir_url(PROJECT_NAME)
57    (url,), _ = mock_get_json_from_url.call_args
58    self.assertEqual(
59        'https://storage.googleapis.com/oss-fuzz-coverage/'
60        'latest_report_info/curl.json', url)
61
62    expected_result = (
63        'https://storage.googleapis.com/oss-fuzz-coverage/systemd/fuzzer_stats/'
64        '20210303')
65    self.assertEqual(result, expected_result)
66
67  def test_get_invalid_project(self):
68    """Tests that passing a bad project returns None."""
69    self.assertIsNone(
70        get_coverage._get_oss_fuzz_fuzzer_stats_dir_url('not-a-proj'))
71
72
73class OSSFuzzCoverageGetTargetCoverageTest(unittest.TestCase):
74  """Tests OSSFuzzCoverage.get_target_coverage."""
75
76  def setUp(self):
77    with mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
78                    return_value=PROJECT_COV_INFO):
79      self.oss_fuzz_coverage = get_coverage.OSSFuzzCoverage(
80          REPO_PATH, PROJECT_NAME)
81
82  @mock.patch('http_utils.get_json_from_url', return_value={})
83  def test_valid_target(self, mock_get_json_from_url):
84    """Tests that a target's coverage report can be downloaded and parsed."""
85    self.oss_fuzz_coverage.get_target_coverage(FUZZ_TARGET)
86    (url,), _ = mock_get_json_from_url.call_args
87    self.assertEqual(
88        'https://storage.googleapis.com/oss-fuzz-coverage/'
89        'curl/fuzzer_stats/20200226/curl_fuzzer.json', url)
90
91  def test_invalid_target(self):
92    """Tests that passing an invalid target coverage report returns None."""
93    self.assertIsNone(
94        self.oss_fuzz_coverage.get_target_coverage(INVALID_TARGET))
95
96  @mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
97              return_value=None)
98  def test_invalid_project_json(self, _):  # pylint: disable=no-self-use
99    """Tests an invalid project JSON results in None being returned."""
100    with pytest.raises(get_coverage.CoverageError):
101      get_coverage.OSSFuzzCoverage(REPO_PATH, PROJECT_NAME)
102
103
104def _get_expected_curl_covered_file_list():
105  """Returns the expected covered file list for
106  FUZZ_TARGET_COV_JSON_FILENAME."""
107  curl_files_list_path = os.path.join(TEST_DATA_PATH,
108                                      'example_curl_file_list.json')
109  with open(curl_files_list_path) as file_handle:
110    return json.loads(file_handle.read())
111
112
113def _get_example_curl_coverage():
114  """Returns the contents of the fuzzer stats JSON file for
115  FUZZ_TARGET_COV_JSON_FILENAME."""
116  with open(os.path.join(TEST_DATA_PATH,
117                         FUZZ_TARGET_COV_JSON_FILENAME)) as file_handle:
118    return json.loads(file_handle.read())
119
120
121class OSSFuzzCoverageGetFilesCoveredByTargetTest(unittest.TestCase):
122  """Tests OSSFuzzCoverage.get_files_covered_by_target."""
123
124  def setUp(self):
125    with mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
126                    return_value=PROJECT_COV_INFO):
127      self.oss_fuzz_coverage = get_coverage.OSSFuzzCoverage(
128          REPO_PATH, PROJECT_NAME)
129
130  def test_valid_target(self):
131    """Tests that covered files can be retrieved from a coverage report."""
132    fuzzer_cov_data = _get_example_curl_coverage()
133    with mock.patch('get_coverage.OSSFuzzCoverage.get_target_coverage',
134                    return_value=fuzzer_cov_data):
135      file_list = self.oss_fuzz_coverage.get_files_covered_by_target(
136          FUZZ_TARGET)
137
138    expected_file_list = _get_expected_curl_covered_file_list()
139    self.assertCountEqual(file_list, expected_file_list)
140
141  def test_invalid_target(self):
142    """Tests passing invalid fuzz target returns None."""
143    self.assertIsNone(
144        self.oss_fuzz_coverage.get_files_covered_by_target(INVALID_TARGET))
145
146
147class FilesystemCoverageGetFilesCoveredByTargetTest(
148    fake_filesystem_unittest.TestCase):
149  """Tests FilesystemCoverage.get_files_covered_by_target."""
150
151  def setUp(self):
152    _fuzzer_cov_data = _get_example_curl_coverage()
153    self._expected_file_list = _get_expected_curl_covered_file_list()
154    self.coverage_path = '/coverage'
155    self.filesystem_coverage = get_coverage.FilesystemCoverage(
156        REPO_PATH, self.coverage_path)
157    self.setUpPyfakefs()
158    self.fs.create_file(os.path.join(self.coverage_path, 'fuzzer_stats',
159                                     FUZZ_TARGET + '.json'),
160                        contents=json.dumps(_fuzzer_cov_data))
161
162  def test_valid_target(self):
163    """Tests that covered files can be retrieved from a coverage report."""
164    file_list = self.filesystem_coverage.get_files_covered_by_target(
165        FUZZ_TARGET)
166    self.assertCountEqual(file_list, self._expected_file_list)
167
168  def test_invalid_target(self):
169    """Tests passing invalid fuzz target returns None."""
170    self.assertIsNone(
171        self.filesystem_coverage.get_files_covered_by_target(INVALID_TARGET))
172
173
174class IsFileCoveredTest(unittest.TestCase):
175  """Tests for is_file_covered."""
176
177  def test_is_file_covered_covered(self):
178    """Tests that is_file_covered returns True for a covered file."""
179    file_coverage = {
180        'filename': '/src/systemd/src/basic/locale-util.c',
181        'summary': {
182            'regions': {
183                'count': 204,
184                'covered': 200,
185                'notcovered': 200,
186                'percent': 98.03
187            }
188        }
189    }
190    self.assertTrue(get_coverage.is_file_covered(file_coverage))
191
192  def test_is_file_covered_not_covered(self):
193    """Tests that is_file_covered returns False for a not covered file."""
194    file_coverage = {
195        'filename': '/src/systemd/src/basic/locale-util.c',
196        'summary': {
197            'regions': {
198                'count': 204,
199                'covered': 0,
200                'notcovered': 0,
201                'percent': 0
202            }
203        }
204    }
205    self.assertFalse(get_coverage.is_file_covered(file_coverage))
206
207
208class GetOssFuzzLatestCovReportInfo(unittest.TestCase):
209  """Tests that _get_oss_fuzz_latest_cov_report_info works as
210  intended."""
211
212  PROJECT = 'project'
213  LATEST_REPORT_INFO_URL = ('https://storage.googleapis.com/oss-fuzz-coverage/'
214                            'latest_report_info/project.json')
215
216  @mock.patch('logging.error')
217  @mock.patch('http_utils.get_json_from_url', return_value={'coverage': 1})
218  def test_get_oss_fuzz_latest_cov_report_info(self, mock_get_json_from_url,
219                                               mock_error):
220    """Tests that _get_oss_fuzz_latest_cov_report_info works as intended."""
221    result = get_coverage._get_oss_fuzz_latest_cov_report_info(self.PROJECT)
222    self.assertEqual(result, {'coverage': 1})
223    mock_error.assert_not_called()
224    mock_get_json_from_url.assert_called_with(self.LATEST_REPORT_INFO_URL)
225
226  @mock.patch('logging.error')
227  @mock.patch('http_utils.get_json_from_url', return_value=None)
228  def test_get_oss_fuzz_latest_cov_report_info_fail(self, _, mock_error):
229    """Tests that _get_oss_fuzz_latest_cov_report_info works as intended when we
230    can't get latest report info."""
231    result = get_coverage._get_oss_fuzz_latest_cov_report_info('project')
232    self.assertIsNone(result)
233    mock_error.assert_called_with(
234        'Could not get the coverage report json from url: %s.',
235        self.LATEST_REPORT_INFO_URL)
236
237
238if __name__ == '__main__':
239  unittest.main()
240