• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Preconfigured checks for Python code.
15
16These checks assume that they are running in a preconfigured Python environment.
17"""
18
19import json
20import logging
21import os
22from pathlib import Path
23import subprocess
24import sys
25from typing import Optional
26
27try:
28    import pw_presubmit
29except ImportError:
30    # Append the pw_presubmit package path to the module search path to allow
31    # running this module without installing the pw_presubmit package.
32    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
33    import pw_presubmit
34
35from pw_env_setup import python_packages
36from pw_presubmit import (
37    build,
38    call,
39    Check,
40    filter_paths,
41    PresubmitContext,
42    PresubmitFailure,
43)
44
45_LOG = logging.getLogger(__name__)
46
47_PYTHON_EXTENSIONS = ('.py', '.gn', '.gni')
48
49_PYTHON_IS_3_9_OR_HIGHER = sys.version_info >= (
50    3,
51    9,
52)
53
54
55@filter_paths(endswith=_PYTHON_EXTENSIONS)
56def gn_python_check(ctx: PresubmitContext):
57    build.gn_gen(ctx)
58    build.ninja(ctx, 'python.tests', 'python.lint')
59
60
61def _transform_lcov_file_paths(lcov_file: Path, repo_root: Path) -> str:
62    """Modify file paths in an lcov file to be relative to the repo root.
63
64    See `man geninfo` for info on the lcov format."""
65
66    lcov_input = lcov_file.read_text()
67    lcov_output = ''
68
69    if not _PYTHON_IS_3_9_OR_HIGHER:
70        return lcov_input
71
72    for line in lcov_input.splitlines():
73        if not line.startswith('SF:'):
74            lcov_output += line + '\n'
75            continue
76
77        # Get the file path after SF:
78        file_string = line[3:].rstrip()
79        source_file_path = Path(file_string)
80
81        # TODO(b/248257406) Remove once we drop support for Python 3.8.
82        def is_relative_to(path: Path, other: Path) -> bool:
83            try:
84                path.relative_to(other)
85                return True
86            except ValueError:
87                return False
88
89        # Attempt to map a generated Python package source file to the root
90        # source tree.
91        # pylint: disable=no-member
92        if not is_relative_to(
93            source_file_path, repo_root  # type: ignore[attr-defined]
94        ):
95            # pylint: enable=no-member
96            source_file_path = repo_root / str(source_file_path).replace(
97                'python/gen/', ''
98            ).replace('py.generated_python_package/', '')
99
100        # If mapping fails don't modify this line.
101        # pylint: disable=no-member
102        if not is_relative_to(
103            source_file_path, repo_root  # type: ignore[attr-defined]
104        ):
105            # pylint: enable=no-member
106            lcov_output += line + '\n'
107            continue
108
109        source_file_path = source_file_path.relative_to(repo_root)
110        lcov_output += f'SF:{source_file_path}\n'
111
112    return lcov_output
113
114
115@filter_paths(endswith=_PYTHON_EXTENSIONS)
116def gn_python_test_coverage(ctx: PresubmitContext):
117    """Run Python tests with coverage and create reports."""
118    build.gn_gen(ctx, pw_build_PYTHON_TEST_COVERAGE=True)
119    build.ninja(ctx, 'python.tests')
120
121    # Find coverage data files
122    coverage_data_files = list(ctx.output_dir.glob('**/*.coverage'))
123    if not coverage_data_files:
124        return
125
126    # Merge coverage data files to out/.coverage
127    call(
128        'coverage',
129        'combine',
130        # Leave existing coverage files in place; by default they are deleted.
131        '--keep',
132        *coverage_data_files,
133        cwd=ctx.output_dir,
134    )
135    combined_data_file = ctx.output_dir / '.coverage'
136    _LOG.info('Coverage data saved to: %s', combined_data_file.resolve())
137
138    # Always ignore generated proto python and setup.py files.
139    coverage_omit_patterns = '--omit=*_pb2.py,*/setup.py'
140
141    # Output coverage percentage summary to the terminal of changed files.
142    changed_python_files = list(
143        str(p) for p in ctx.paths if str(p).endswith('.py')
144    )
145    report_args = [
146        'coverage',
147        'report',
148        '--ignore-errors',
149        coverage_omit_patterns,
150    ]
151    report_args += changed_python_files
152    subprocess.run(report_args, check=False, cwd=ctx.output_dir)
153
154    # Generate a json report
155    call('coverage', 'lcov', coverage_omit_patterns, cwd=ctx.output_dir)
156    lcov_data_file = ctx.output_dir / 'coverage.lcov'
157    lcov_data_file.write_text(
158        _transform_lcov_file_paths(lcov_data_file, repo_root=ctx.root)
159    )
160    _LOG.info('Coverage lcov saved to: %s', lcov_data_file.resolve())
161
162    # Generate an html report
163    call('coverage', 'html', coverage_omit_patterns, cwd=ctx.output_dir)
164    html_report = ctx.output_dir / 'htmlcov' / 'index.html'
165    _LOG.info('Coverage html report saved to: %s', html_report.resolve())
166
167
168@filter_paths(endswith=_PYTHON_EXTENSIONS + ('.pylintrc',))
169def gn_python_lint(ctx: pw_presubmit.PresubmitContext) -> None:
170    build.gn_gen(ctx)
171    build.ninja(ctx, 'python.lint')
172
173
174@Check
175def check_python_versions(ctx: PresubmitContext):
176    """Checks that the list of installed packages is as expected."""
177
178    build.gn_gen(ctx)
179    constraint_file: Optional[str] = None
180    requirement_file: Optional[str] = None
181    try:
182        for arg in build.get_gn_args(ctx.output_dir):
183            if arg['name'] == 'pw_build_PIP_CONSTRAINTS':
184                constraint_file = json.loads(arg['current']['value'])[0].strip(
185                    '/'
186                )
187            if arg['name'] == 'pw_build_PIP_REQUIREMENTS':
188                requirement_file = json.loads(arg['current']['value'])[0].strip(
189                    '/'
190                )
191    except json.JSONDecodeError:
192        _LOG.warning('failed to parse GN args json')
193        return
194
195    if not constraint_file:
196        _LOG.warning('could not find pw_build_PIP_CONSTRAINTS GN arg')
197        return
198    ignored_requirements_arg = None
199    if requirement_file:
200        ignored_requirements_arg = [(ctx.root / requirement_file)]
201
202    if (
203        python_packages.diff(
204            expected=(ctx.root / constraint_file),
205            ignore_requirements_file=ignored_requirements_arg,
206        )
207        != 0
208    ):
209        raise PresubmitFailure
210