• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Preconfigured checks for Python code.
15
16These checks assume that they are running in a preconfigured Python environment.
17"""
18
19import json
20import logging
21import os
22from pathlib import Path
23import subprocess
24import sys
25from typing import Optional
26
27try:
28    import pw_presubmit
29except ImportError:
30    # Append the pw_presubmit package path to the module search path to allow
31    # running this module without installing the pw_presubmit package.
32    sys.path.append(os.path.dirname(os.path.dirname(
33        os.path.abspath(__file__))))
34    import pw_presubmit
35
36from pw_env_setup import python_packages
37from pw_presubmit import (
38    build,
39    call,
40    Check,
41    filter_paths,
42    PresubmitContext,
43    PresubmitFailure,
44)
45
46_LOG = logging.getLogger(__name__)
47
48_PYTHON_EXTENSIONS = ('.py', '.gn', '.gni')
49
50_PYTHON_IS_3_9_OR_HIGHER = sys.version_info >= (
51    3,
52    9,
53)
54
55
56@filter_paths(endswith=_PYTHON_EXTENSIONS)
57def gn_python_check(ctx: PresubmitContext):
58    build.gn_gen(ctx.root, ctx.output_dir)
59    build.ninja(ctx.output_dir, 'python.tests', 'python.lint')
60
61
62def _transform_lcov_file_paths(lcov_file: Path, repo_root: Path) -> str:
63    """Modify file paths in an lcov file to be relative to the repo root.
64
65    See `man geninfo` for info on the lcov format."""
66
67    lcov_input = lcov_file.read_text()
68    lcov_output = ''
69
70    if not _PYTHON_IS_3_9_OR_HIGHER:
71        return lcov_input
72
73    for line in lcov_input.splitlines():
74        if not line.startswith('SF:'):
75            lcov_output += line + '\n'
76            continue
77
78        # Get the file path after SF:
79        file_string = line[3:].rstrip()
80        source_file_path = Path(file_string)
81
82        # Attempt to map a generated Python package source file to the root
83        # source tree.
84        # pylint: disable=no-member
85        if not source_file_path.is_relative_to(  # type: ignore[attr-defined]
86                repo_root):
87            # pylint: enable=no-member
88            source_file_path = repo_root / str(source_file_path).replace(
89                'python/gen/', '').replace('py.generated_python_package/', '')
90
91        # If mapping fails don't modify this line.
92        # pylint: disable=no-member
93        if not source_file_path.is_relative_to(  # type: ignore[attr-defined]
94                repo_root):
95            # pylint: enable=no-member
96            lcov_output += line + '\n'
97            continue
98
99        source_file_path = source_file_path.relative_to(repo_root)
100        lcov_output += f'SF:{source_file_path}\n'
101
102    return lcov_output
103
104
105@filter_paths(endswith=_PYTHON_EXTENSIONS)
106def gn_python_test_coverage(ctx: PresubmitContext):
107    """Run Python tests with coverage and create reports."""
108    build.gn_gen(ctx.root, ctx.output_dir, pw_build_PYTHON_TEST_COVERAGE=True)
109    build.ninja(ctx.output_dir, 'python.tests')
110
111    # Find coverage data files
112    coverage_data_files = list(ctx.output_dir.glob('**/*.coverage'))
113    if not coverage_data_files:
114        return
115
116    # Merge coverage data files to out/.coverage
117    call(
118        'coverage',
119        'combine',
120        # Leave existing coverage files in place; by default they are deleted.
121        '--keep',
122        *coverage_data_files,
123        cwd=ctx.output_dir)
124    combined_data_file = ctx.output_dir / '.coverage'
125    _LOG.info('Coverage data saved to: %s', combined_data_file.resolve())
126
127    # Always ignore generated proto python and setup.py files.
128    coverage_omit_patterns = '--omit=*_pb2.py,*/setup.py'
129
130    # Output coverage percentage summary to the terminal of changed files.
131    changed_python_files = list(
132        str(p) for p in ctx.paths if str(p).endswith('.py'))
133    report_args = [
134        'coverage',
135        'report',
136        '--ignore-errors',
137        coverage_omit_patterns,
138    ]
139    report_args += changed_python_files
140    subprocess.run(report_args, check=False, cwd=ctx.output_dir)
141
142    # Generate a json report
143    call('coverage', 'lcov', coverage_omit_patterns, cwd=ctx.output_dir)
144    lcov_data_file = ctx.output_dir / 'coverage.lcov'
145    lcov_data_file.write_text(
146        _transform_lcov_file_paths(lcov_data_file, repo_root=ctx.root))
147    _LOG.info('Coverage lcov saved to: %s', lcov_data_file.resolve())
148
149    # Generate an html report
150    call('coverage', 'html', coverage_omit_patterns, cwd=ctx.output_dir)
151    html_report = ctx.output_dir / 'htmlcov' / 'index.html'
152    _LOG.info('Coverage html report saved to: %s', html_report.resolve())
153
154
155@filter_paths(endswith=_PYTHON_EXTENSIONS + ('.pylintrc', ))
156def gn_python_lint(ctx: pw_presubmit.PresubmitContext) -> None:
157    build.gn_gen(ctx.root, ctx.output_dir)
158    build.ninja(ctx.output_dir, 'python.lint')
159
160
161@Check
162def check_python_versions(ctx: PresubmitContext):
163    """Checks that the list of installed packages is as expected."""
164
165    build.gn_gen(ctx.root, ctx.output_dir)
166    constraint_file: Optional[str] = None
167    try:
168        for arg in build.get_gn_args(ctx.output_dir):
169            if arg['name'] == 'pw_build_PIP_CONSTRAINTS':
170                constraint_file = json.loads(
171                    arg['current']['value'])[0].strip('/')
172    except json.JSONDecodeError:
173        _LOG.warning('failed to parse GN args json')
174        return
175
176    if not constraint_file:
177        _LOG.warning('could not find pw_build_PIP_CONSTRAINTS GN arg')
178        return
179
180    with (ctx.root / constraint_file).open('r') as ins:
181        if python_packages.diff(ins) != 0:
182            raise PresubmitFailure
183