1#!/usr/bin/env python3 2 3# Copyright 2020 The Pigweed Authors 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may not 6# use this file except in compliance with the License. You may obtain a copy of 7# the License at 8# 9# https://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14# License for the specific language governing permissions and limitations under 15# the License. 16"""Runs the local presubmit checks for the Pigweed repository.""" 17 18import argparse 19import json 20import logging 21import os 22from pathlib import Path 23import re 24import subprocess 25import sys 26from typing import Sequence, IO, Tuple, Optional, Callable, List 27 28try: 29 import pw_presubmit 30except ImportError: 31 # Append the pw_presubmit package path to the module search path to allow 32 # running this module without installing the pw_presubmit package. 33 sys.path.append(os.path.dirname(os.path.dirname( 34 os.path.abspath(__file__)))) 35 import pw_presubmit 36 37import pw_package.pigweed_packages 38 39from pw_presubmit import ( 40 build, 41 cli, 42 cpp_checks, 43 format_code, 44 git_repo, 45 call, 46 filter_paths, 47 inclusive_language, 48 plural, 49 PresubmitContext, 50 PresubmitFailure, 51 Programs, 52 python_checks, 53) 54from pw_presubmit.install_hook import install_hook 55 56_LOG = logging.getLogger(__name__) 57 58pw_package.pigweed_packages.initialize() 59 60# Trigger builds if files with these extensions change. 61_BUILD_EXTENSIONS = ('.py', '.rst', '.gn', '.gni', 62 *format_code.C_FORMAT.extensions) 63 64 65def _at_all_optimization_levels(target): 66 for level in ('debug', 'size_optimized', 'speed_optimized'): 67 yield f'{target}_{level}' 68 69 70# 71# Build presubmit checks 72# 73def gn_clang_build(ctx: PresubmitContext): 74 build.gn_gen(ctx.root, 75 ctx.output_dir, 76 pw_RUN_INTEGRATION_TESTS=(sys.platform != 'win32')) 77 build.ninja(ctx.output_dir, *_at_all_optimization_levels('host_clang')) 78 79 80@filter_paths(endswith=_BUILD_EXTENSIONS) 81def gn_gcc_build(ctx: PresubmitContext): 82 build.gn_gen(ctx.root, ctx.output_dir) 83 build.ninja(ctx.output_dir, *_at_all_optimization_levels('host_gcc')) 84 85 86_HOST_COMPILER = 'gcc' if sys.platform == 'win32' else 'clang' 87 88 89def gn_host_build(ctx: PresubmitContext): 90 build.gn_gen(ctx.root, ctx.output_dir) 91 build.ninja(ctx.output_dir, 92 *_at_all_optimization_levels(f'host_{_HOST_COMPILER}')) 93 94 95@filter_paths(endswith=_BUILD_EXTENSIONS) 96def gn_quick_build_check(ctx: PresubmitContext): 97 """Checks the state of the GN build by running gn gen and gn check.""" 98 build.gn_gen(ctx.root, ctx.output_dir) 99 100 101@filter_paths(endswith=_BUILD_EXTENSIONS) 102def gn_full_build_check(ctx: PresubmitContext): 103 build.gn_gen(ctx.root, ctx.output_dir) 104 build.ninja(ctx.output_dir, *_at_all_optimization_levels('stm32f429i'), 105 *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), 106 'python.tests', 'python.lint', 'docs', 'fuzzers', 107 'pw_env_setup:build_pigweed_python_source_tree') 108 109 110@filter_paths(endswith=_BUILD_EXTENSIONS) 111def gn_full_qemu_check(ctx: PresubmitContext): 112 build.gn_gen(ctx.root, ctx.output_dir) 113 build.ninja( 114 ctx.output_dir, 115 *_at_all_optimization_levels('qemu_gcc'), 116 # TODO(pwbug/321) Re-enable clang. 117 # *_at_all_optimization_levels('qemu_clang'), 118 ) 119 120 121@filter_paths(endswith=_BUILD_EXTENSIONS) 122def gn_arm_build(ctx: PresubmitContext): 123 build.gn_gen(ctx.root, ctx.output_dir) 124 build.ninja(ctx.output_dir, *_at_all_optimization_levels('stm32f429i')) 125 126 127@filter_paths(endswith=_BUILD_EXTENSIONS) 128def stm32f429i(ctx: PresubmitContext): 129 build.gn_gen(ctx.root, ctx.output_dir, pw_use_test_server=True) 130 with build.test_server('stm32f429i_disc1_test_server', ctx.output_dir): 131 build.ninja(ctx.output_dir, *_at_all_optimization_levels('stm32f429i')) 132 133 134@filter_paths(endswith=_BUILD_EXTENSIONS) 135def gn_boringssl_build(ctx: PresubmitContext): 136 build.install_package(ctx.package_root, 'boringssl') 137 build.gn_gen(ctx.root, 138 ctx.output_dir, 139 dir_pw_third_party_boringssl='"{}"'.format(ctx.package_root / 140 'boringssl')) 141 build.ninja( 142 ctx.output_dir, 143 *_at_all_optimization_levels('stm32f429i'), 144 *_at_all_optimization_levels('host_clang'), 145 ) 146 147 148@filter_paths(endswith=_BUILD_EXTENSIONS) 149def gn_nanopb_build(ctx: PresubmitContext): 150 build.install_package(ctx.package_root, 'nanopb') 151 build.gn_gen(ctx.root, 152 ctx.output_dir, 153 dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root / 154 'nanopb')) 155 build.ninja( 156 ctx.output_dir, 157 *_at_all_optimization_levels('stm32f429i'), 158 *_at_all_optimization_levels('host_clang'), 159 ) 160 161 162@filter_paths(endswith=_BUILD_EXTENSIONS) 163def gn_crypto_mbedtls_build(ctx: PresubmitContext): 164 build.install_package(ctx.package_root, 'mbedtls') 165 build.gn_gen( 166 ctx.root, 167 ctx.output_dir, 168 dir_pw_third_party_mbedtls='"{}"'.format(ctx.package_root / 'mbedtls'), 169 pw_crypto_SHA256_BACKEND='"{}"'.format(ctx.root / 170 'pw_crypto:sha256_mbedtls'), 171 pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root / 172 'pw_crypto:ecdsa_mbedtls')) 173 build.ninja(ctx.output_dir) 174 175 176@filter_paths(endswith=_BUILD_EXTENSIONS) 177def gn_crypto_boringssl_build(ctx: PresubmitContext): 178 build.install_package(ctx.package_root, 'boringssl') 179 build.gn_gen( 180 ctx.root, 181 ctx.output_dir, 182 dir_pw_third_party_boringssl='"{}"'.format(ctx.package_root / 183 'boringssl'), 184 pw_crypto_SHA256_BACKEND='"{}"'.format(ctx.root / 185 'pw_crypto:sha256_boringssl'), 186 pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root / 187 'pw_crypto:ecdsa_boringssl'), 188 ) 189 build.ninja(ctx.output_dir) 190 191 192@filter_paths(endswith=_BUILD_EXTENSIONS) 193def gn_crypto_micro_ecc_build(ctx: PresubmitContext): 194 build.install_package(ctx.package_root, 'micro-ecc') 195 build.gn_gen( 196 ctx.root, 197 ctx.output_dir, 198 dir_pw_third_party_micro_ecc='"{}"'.format(ctx.package_root / 199 'micro-ecc'), 200 pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root / 201 'pw_crypto:ecdsa_uecc'), 202 ) 203 build.ninja(ctx.output_dir) 204 205 206@filter_paths(endswith=_BUILD_EXTENSIONS) 207def gn_teensy_build(ctx: PresubmitContext): 208 build.install_package(ctx.package_root, 'teensy') 209 build.gn_gen(ctx.root, 210 ctx.output_dir, 211 pw_arduino_build_CORE_PATH='"{}"'.format(str( 212 ctx.package_root)), 213 pw_arduino_build_CORE_NAME='teensy', 214 pw_arduino_build_PACKAGE_NAME='teensy/avr', 215 pw_arduino_build_BOARD='teensy40') 216 build.ninja(ctx.output_dir, *_at_all_optimization_levels('arduino')) 217 218 219@filter_paths(endswith=_BUILD_EXTENSIONS) 220def gn_software_update_build(ctx: PresubmitContext): 221 build.install_package(ctx.package_root, 'nanopb') 222 build.install_package(ctx.package_root, 'protobuf') 223 build.install_package(ctx.package_root, 'mbedtls') 224 build.install_package(ctx.package_root, 'micro-ecc') 225 build.gn_gen( 226 ctx.root, 227 ctx.output_dir, 228 dir_pw_third_party_protobuf='"{}"'.format(ctx.package_root / 229 'protobuf'), 230 dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root / 'nanopb'), 231 dir_pw_third_party_micro_ecc='"{}"'.format(ctx.package_root / 232 'micro-ecc'), 233 pw_crypto_ECDSA_BACKEND='"{}"'.format(ctx.root / 234 'pw_crypto:ecdsa_uecc'), 235 dir_pw_third_party_mbedtls='"{}"'.format(ctx.package_root / 'mbedtls'), 236 pw_crypto_SHA256_BACKEND='"{}"'.format(ctx.root / 237 'pw_crypto:sha256_mbedtls')) 238 build.ninja( 239 ctx.output_dir, 240 *_at_all_optimization_levels('host_clang'), 241 ) 242 243 244@filter_paths(endswith=_BUILD_EXTENSIONS) 245def gn_pw_system_demo_build(ctx: PresubmitContext): 246 build.install_package(ctx.package_root, 'freertos') 247 build.install_package(ctx.package_root, 'nanopb') 248 build.install_package(ctx.package_root, 'stm32cube_f4') 249 build.gn_gen( 250 ctx.root, 251 ctx.output_dir, 252 dir_pw_third_party_freertos='"{}"'.format(ctx.package_root / 253 'freertos'), 254 dir_pw_third_party_nanopb='"{}"'.format(ctx.package_root / 'nanopb'), 255 dir_pw_third_party_stm32cube_f4='"{}"'.format(ctx.package_root / 256 'stm32cube_f4'), 257 ) 258 build.ninja(ctx.output_dir, 'pw_system_demo') 259 260 261@filter_paths(endswith=_BUILD_EXTENSIONS) 262def gn_qemu_build(ctx: PresubmitContext): 263 build.gn_gen(ctx.root, ctx.output_dir) 264 build.ninja(ctx.output_dir, *_at_all_optimization_levels('qemu_gcc')) 265 266 267@filter_paths(endswith=_BUILD_EXTENSIONS) 268def gn_qemu_clang_build(ctx: PresubmitContext): 269 build.gn_gen(ctx.root, ctx.output_dir) 270 build.ninja(ctx.output_dir, *_at_all_optimization_levels('qemu_clang')) 271 272 273def gn_docs_build(ctx: PresubmitContext): 274 build.gn_gen(ctx.root, ctx.output_dir) 275 build.ninja(ctx.output_dir, 'docs') 276 277 278def gn_host_tools(ctx: PresubmitContext): 279 build.gn_gen(ctx.root, ctx.output_dir) 280 build.ninja(ctx.output_dir, 'host_tools') 281 282 283@filter_paths(endswith=format_code.C_FORMAT.extensions) 284def oss_fuzz_build(ctx: PresubmitContext): 285 build.gn_gen(ctx.root, ctx.output_dir, pw_toolchain_OSS_FUZZ_ENABLED=True) 286 build.ninja(ctx.output_dir, "fuzzers") 287 288 289def _run_cmake(ctx: PresubmitContext, toolchain='host_clang') -> None: 290 build.install_package(ctx.package_root, 'nanopb') 291 292 env = None 293 if 'clang' in toolchain: 294 env = build.env_with_clang_vars() 295 296 toolchain_path = ctx.root / 'pw_toolchain' / toolchain / 'toolchain.cmake' 297 build.cmake(ctx.root, 298 ctx.output_dir, 299 f'-DCMAKE_TOOLCHAIN_FILE={toolchain_path}', 300 '-DCMAKE_EXPORT_COMPILE_COMMANDS=1', 301 f'-Ddir_pw_third_party_nanopb={ctx.package_root / "nanopb"}', 302 '-Dpw_third_party_nanopb_ADD_SUBDIRECTORY=ON', 303 env=env) 304 305 306@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake', 307 'CMakeLists.txt')) 308def cmake_clang(ctx: PresubmitContext): 309 _run_cmake(ctx, toolchain='host_clang') 310 build.ninja(ctx.output_dir, 'pw_apps', 'pw_run_tests.modules') 311 312 313@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.cmake', 314 'CMakeLists.txt')) 315def cmake_gcc(ctx: PresubmitContext): 316 _run_cmake(ctx, toolchain='host_gcc') 317 build.ninja(ctx.output_dir, 'pw_apps', 'pw_run_tests.modules') 318 319 320# TODO(pwbug/180): Slowly add modules here that work with bazel until all 321# modules are added. Then replace with //... 322_MODULES_THAT_BUILD_WITH_BAZEL = [ 323 '//pw_allocator/...', 324 '//pw_analog/...', 325 '//pw_assert/...', 326 '//pw_assert_basic/...', 327 '//pw_assert_log/...', 328 '//pw_base64/...', 329 '//pw_bloat/...', 330 '//pw_build/...', 331 '//pw_checksum/...', 332 '//pw_chrono_embos/...', 333 '//pw_chrono_freertos/...', 334 '//pw_chrono_stl/...', 335 '//pw_chrono_threadx/...', 336 '//pw_cli/...', 337 '//pw_containers/...', 338 '//pw_cpu_exception/...', 339 '//pw_docgen/...', 340 '//pw_doctor/...', 341 '//pw_env_setup/...', 342 '//pw_fuzzer/...', 343 '//pw_hex_dump/...', 344 '//pw_i2c/...', 345 '//pw_interrupt/...', 346 '//pw_interrupt_cortex_m/...', 347 '//pw_libc/...', 348 '//pw_log/...', 349 '//pw_log_basic/...', 350 '//pw_malloc/...', 351 '//pw_malloc_freelist/...', 352 '//pw_multisink/...', 353 '//pw_polyfill/...', 354 '//pw_preprocessor/...', 355 '//pw_protobuf/...', 356 '//pw_protobuf_compiler/...', 357 '//pw_random/...', 358 '//pw_result/...', 359 '//pw_rpc/...', 360 '//pw_span/...', 361 '//pw_status/...', 362 '//pw_stream/...', 363 '//pw_string/...', 364 '//pw_sync_baremetal/...', 365 '//pw_sync_embos/...', 366 '//pw_sync_freertos/...', 367 '//pw_sync_stl/...', 368 '//pw_sync_threadx/...', 369 '//pw_sys_io/...', 370 '//pw_sys_io_baremetal_lm3s6965evb/...', 371 '//pw_sys_io_baremetal_stm32f429/...', 372 '//pw_sys_io_stdio/...', 373 '//pw_thread_stl/...', 374 '//pw_tool/...', 375 '//pw_toolchain/...', 376 '//pw_transfer/...', 377 '//pw_unit_test/...', 378 '//pw_varint/...', 379 '//pw_web_ui/...', 380] 381 382# TODO(pwbug/180): Slowly add modules here that work with bazel until all 383# modules are added. Then replace with //... 384_MODULES_THAT_TEST_WITH_BAZEL = [ 385 '//pw_allocator/...', 386 '//pw_analog/...', 387 '//pw_assert/...', 388 '//pw_base64/...', 389 '//pw_checksum/...', 390 '//pw_cli/...', 391 '//pw_containers/...', 392 '//pw_hex_dump/...', 393 '//pw_i2c/...', 394 '//pw_libc/...', 395 '//pw_log/...', 396 '//pw_multisink/...', 397 '//pw_polyfill/...', 398 '//pw_preprocessor/...', 399 '//pw_protobuf/...', 400 '//pw_protobuf_compiler/...', 401 '//pw_random/...', 402 '//pw_result/...', 403 '//pw_rpc/...', 404 '//pw_span/...', 405 '//pw_status/...', 406 '//pw_stream/...', 407 '//pw_string/...', 408 '//pw_thread_stl/...', 409 '//pw_unit_test/...', 410 '//pw_varint/...', 411 '//:buildifier_test', 412] 413 414 415@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.bazel', '.bzl', 416 'BUILD')) 417def bazel_test(ctx: PresubmitContext) -> None: 418 """Runs bazel test on each bazel compatible module""" 419 build.bazel(ctx, 'test', *_MODULES_THAT_TEST_WITH_BAZEL, 420 '--test_output=errors') 421 422 423@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.bazel', '.bzl', 424 'BUILD')) 425def bazel_build(ctx: PresubmitContext) -> None: 426 """Runs Bazel build on each Bazel compatible module.""" 427 build.bazel(ctx, 'build', *_MODULES_THAT_BUILD_WITH_BAZEL) 428 429 430# 431# General presubmit checks 432# 433 434 435def _clang_system_include_paths(lang: str) -> List[str]: 436 """Generate default system header paths. 437 438 Returns the list of system include paths used by the host 439 clang installation. 440 """ 441 # Dump system include paths with preprocessor verbose. 442 command = [ 443 'clang++', '-Xpreprocessor', '-v', '-x', f'{lang}', f'{os.devnull}', 444 '-fsyntax-only' 445 ] 446 process = subprocess.run(command, 447 check=True, 448 stdout=subprocess.PIPE, 449 stderr=subprocess.STDOUT) 450 451 # Parse the command output to retrieve system include paths. 452 # The paths are listed one per line. 453 output = process.stdout.decode(errors='backslashreplace') 454 include_paths: List[str] = [] 455 for line in output.splitlines(): 456 path = line.strip() 457 if os.path.exists(path): 458 include_paths.append(f'-isystem{path}') 459 460 return include_paths 461 462 463def edit_compile_commands(in_path: Path, out_path: Path, 464 func: Callable[[str, str, str], str]) -> None: 465 """Edit the selected compile command file. 466 467 Calls the input callback on all triplets (file, directory, command) in 468 the input compile commands database. The return value replaces the old 469 compile command in the output database. 470 """ 471 with open(in_path) as in_file: 472 compile_commands = json.load(in_file) 473 for item in compile_commands: 474 item['command'] = func(item['file'], item['directory'], 475 item['command']) 476 with open(out_path, 'w') as out_file: 477 json.dump(compile_commands, out_file, indent=2) 478 479 480# The first line must be regex because of the '20\d\d' date 481COPYRIGHT_FIRST_LINE = r'Copyright 20\d\d The Pigweed Authors' 482COPYRIGHT_COMMENTS = r'(#|//| \*|REM|::)' 483COPYRIGHT_BLOCK_COMMENTS = ( 484 # HTML comments 485 (r'<!--', r'-->'), 486 # Jinja comments 487 (r'{#', r'#}'), 488) 489 490COPYRIGHT_FIRST_LINE_EXCEPTIONS = ( 491 '#!', 492 '/*', 493 '@echo off', 494 '# -*-', 495 ':', 496) 497 498COPYRIGHT_LINES = tuple("""\ 499 500Licensed under the Apache License, Version 2.0 (the "License"); you may not 501use this file except in compliance with the License. You may obtain a copy of 502the License at 503 504 https://www.apache.org/licenses/LICENSE-2.0 505 506Unless required by applicable law or agreed to in writing, software 507distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 508WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 509License for the specific language governing permissions and limitations under 510the License. 511""".splitlines()) 512 513_EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = ( 514 # Configuration 515 r'^(?:.+/)?\..+$', 516 r'\bPW_PLUGINS$', 517 r'\bconstraint.list$', 518 # Metadata 519 r'^docker/tag$', 520 r'\bAUTHORS$', 521 r'\bLICENSE$', 522 r'\bOWNERS$', 523 r'\bPIGWEED_MODULES$', 524 r'\brequirements.txt$', 525 r'\bgo.(mod|sum)$', 526 r'\bpackage.json$', 527 r'\byarn.lock$', 528 # Data files 529 r'\.elf$', 530 r'\.gif$', 531 r'\.jpg$', 532 r'\.json$', 533 r'\.png$', 534 r'\.svg$', 535 r'\.xml$', 536 # Documentation 537 r'\.md$', 538 r'\.rst$', 539 # Generated protobuf files 540 r'\.pb\.h$', 541 r'\.pb\.c$', 542 r'\_pb2.pyi?$', 543 # Diff/Patch files 544 r'\.diff$', 545 r'\.patch$', 546) 547 548 549def match_block_comment_start(line: str) -> Optional[str]: 550 """Matches the start of a block comment and returns the end.""" 551 for block_comment in COPYRIGHT_BLOCK_COMMENTS: 552 if re.match(block_comment[0], line): 553 # Return the end of the block comment 554 return block_comment[1] 555 return None 556 557 558def copyright_read_first_line( 559 file: IO) -> Tuple[Optional[str], Optional[str], Optional[str]]: 560 """Reads the file until it reads a valid first copyright line. 561 562 Returns (comment, block_comment, line). comment and block_comment are 563 mutually exclusive and refer to the comment character sequence and whether 564 they form a block comment or a line comment. line is the first line of 565 the copyright, and is used for error reporting. 566 """ 567 line = file.readline() 568 first_line_matcher = re.compile(COPYRIGHT_COMMENTS + ' ' + 569 COPYRIGHT_FIRST_LINE) 570 while line: 571 end_block_comment = match_block_comment_start(line) 572 if end_block_comment: 573 next_line = file.readline() 574 copyright_line = re.match(COPYRIGHT_FIRST_LINE, next_line) 575 if not copyright_line: 576 return (None, None, line) 577 return (None, end_block_comment, line) 578 579 first_line = first_line_matcher.match(line) 580 if first_line: 581 return (first_line.group(1), None, line) 582 583 if (line.strip() 584 and not line.startswith(COPYRIGHT_FIRST_LINE_EXCEPTIONS)): 585 return (None, None, line) 586 587 line = file.readline() 588 return (None, None, None) 589 590 591@filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE) 592def copyright_notice(ctx: PresubmitContext): 593 """Checks that the Pigweed copyright notice is present.""" 594 errors = [] 595 596 for path in ctx.paths: 597 598 if path.stat().st_size == 0: 599 continue # Skip empty files 600 601 if path.is_dir(): 602 continue # Skip submodules which are included in ctx.paths. 603 604 with path.open() as file: 605 (comment, end_block_comment, 606 line) = copyright_read_first_line(file) 607 608 if not line: 609 _LOG.warning('%s: invalid first line', path) 610 errors.append(path) 611 continue 612 613 if not (comment or end_block_comment): 614 _LOG.warning('%s: invalid first line %r', path, line) 615 errors.append(path) 616 continue 617 618 if end_block_comment: 619 expected_lines = COPYRIGHT_LINES + (end_block_comment, ) 620 else: 621 expected_lines = COPYRIGHT_LINES 622 623 for expected, actual in zip(expected_lines, file): 624 if end_block_comment: 625 expected_line = expected + '\n' 626 elif comment: 627 expected_line = (comment + ' ' + expected).rstrip() + '\n' 628 629 if expected_line != actual: 630 _LOG.warning(' bad line: %r', actual) 631 _LOG.warning(' expected: %r', expected_line) 632 errors.append(path) 633 break 634 635 if errors: 636 _LOG.warning('%s with a missing or incorrect copyright notice:\n%s', 637 plural(errors, 'file'), '\n'.join(str(e) for e in errors)) 638 raise PresubmitFailure 639 640 641_BAZEL_SOURCES_IN_BUILD = tuple(format_code.C_FORMAT.extensions) 642_GN_SOURCES_IN_BUILD = ('setup.cfg', '.toml', '.rst', '.py', 643 *_BAZEL_SOURCES_IN_BUILD) 644 645 646@filter_paths(endswith=(*_GN_SOURCES_IN_BUILD, 'BUILD', '.bzl', '.gn', '.gni'), 647 exclude=['zephyr.*/', 'android.*/']) 648def source_is_in_build_files(ctx: PresubmitContext): 649 """Checks that source files are in the GN and Bazel builds.""" 650 missing = build.check_builds_for_files( 651 _BAZEL_SOURCES_IN_BUILD, 652 _GN_SOURCES_IN_BUILD, 653 ctx.paths, 654 bazel_dirs=[ctx.root], 655 gn_build_files=git_repo.list_files(pathspecs=['BUILD.gn', '*BUILD.gn'], 656 repo_path=ctx.root)) 657 658 if missing: 659 _LOG.warning( 660 'All source files must appear in BUILD and BUILD.gn files') 661 raise PresubmitFailure 662 663 _run_cmake(ctx) 664 cmake_missing = build.check_compile_commands_for_files( 665 ctx.output_dir / 'compile_commands.json', 666 (f for f in ctx.paths if f.suffix in ('.c', '.cc'))) 667 if cmake_missing: 668 _LOG.warning('The CMake build is missing %d files', len(cmake_missing)) 669 _LOG.warning('Files missing from CMake:\n%s', 670 '\n'.join(str(f) for f in cmake_missing)) 671 # TODO(hepler): Many files are missing from the CMake build. Make this 672 # check an error when the missing files are fixed. 673 # raise PresubmitFailure 674 675 676def build_env_setup(ctx: PresubmitContext): 677 if 'PW_CARGO_SETUP' not in os.environ: 678 _LOG.warning( 679 'Skipping build_env_setup since PW_CARGO_SETUP is not set') 680 return 681 682 tmpl = ctx.root.joinpath('pw_env_setup', 'py', 'pyoxidizer.bzl.tmpl') 683 out = ctx.output_dir.joinpath('pyoxidizer.bzl') 684 685 with open(tmpl, 'r') as ins: 686 cfg = ins.read().replace('${PW_ROOT}', str(ctx.root)) 687 with open(out, 'w') as outs: 688 outs.write(cfg) 689 690 call('pyoxidizer', 'build', cwd=ctx.output_dir) 691 692 693def commit_message_format(_: PresubmitContext): 694 """Checks that the top commit's message is correctly formatted.""" 695 lines = git_repo.commit_message().splitlines() 696 697 # Show limits and current commit message in log. 698 _LOG.debug('%-25s%+25s%+22s', 'Line limits', '72|', '72|') 699 for line in lines: 700 _LOG.debug(line) 701 702 # Ignore Gerrit-generated reverts. 703 if ('Revert' in lines[0] 704 and 'This reverts commit ' in git_repo.commit_message() 705 and 'Reason for revert: ' in git_repo.commit_message()): 706 _LOG.warning('Ignoring apparent Gerrit-generated revert') 707 return 708 709 if not lines: 710 _LOG.error('The commit message is too short!') 711 raise PresubmitFailure 712 713 errors = 0 714 715 if len(lines[0]) > 72: 716 _LOG.warning("The commit message's first line must be no longer than " 717 '72 characters.') 718 _LOG.warning('The first line is %d characters:\n %s', len(lines[0]), 719 lines[0]) 720 errors += 1 721 722 if lines[0].endswith('.'): 723 _LOG.warning( 724 "The commit message's first line must not end with a period:\n %s", 725 lines[0]) 726 errors += 1 727 728 if len(lines) > 1 and lines[1]: 729 _LOG.warning("The commit message's second line must be blank.") 730 _LOG.warning('The second line has %d characters:\n %s', len(lines[1]), 731 lines[1]) 732 errors += 1 733 734 # Check that the lines are 72 characters or less, but skip any lines that 735 # might possibly have a URL, path, or metadata in them. Also skip any lines 736 # with non-ASCII characters. 737 for i, line in enumerate(lines[2:], 3): 738 if any(c in line for c in ':/>') or not line.isascii(): 739 continue 740 741 if len(line) > 72: 742 _LOG.warning( 743 'Commit message lines must be no longer than 72 characters.') 744 _LOG.warning('Line %d has %d characters:\n %s', i, len(line), 745 line) 746 errors += 1 747 748 if errors: 749 _LOG.error('Found %s in the commit message', plural(errors, 'error')) 750 raise PresubmitFailure 751 752 753@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.py')) 754def static_analysis(ctx: PresubmitContext): 755 """Runs all available static analysis tools.""" 756 build.gn_gen(ctx.root, ctx.output_dir) 757 build.ninja(ctx.output_dir, 'python.lint', 'static_analysis') 758 759 760def renode_check(ctx: PresubmitContext): 761 """Placeholder for future check.""" 762 _LOG.info('%s %s', ctx.root, ctx.output_dir) 763 764 765# 766# Presubmit check programs 767# 768 769OTHER_CHECKS = ( 770 cpp_checks.all_sanitizers(), 771 # Build that attempts to duplicate the build OSS-Fuzz does. Currently 772 # failing. 773 oss_fuzz_build, 774 # TODO(pwbug/346): Enable all Bazel tests when they're fixed. 775 bazel_test, 776 cmake_clang, 777 cmake_gcc, 778 gn_boringssl_build, 779 build.gn_gen_check, 780 gn_nanopb_build, 781 gn_crypto_mbedtls_build, 782 gn_crypto_boringssl_build, 783 gn_crypto_micro_ecc_build, 784 gn_software_update_build, 785 gn_full_build_check, 786 gn_full_qemu_check, 787 gn_clang_build, 788 gn_gcc_build, 789 gn_pw_system_demo_build, 790 renode_check, 791 stm32f429i, 792) 793 794_LINTFORMAT = ( 795 commit_message_format, 796 copyright_notice, 797 format_code.presubmit_checks(), 798 inclusive_language.inclusive_language.with_filter( 799 exclude=(r'\byarn.lock$', )), 800 cpp_checks.pragma_once, 801 build.bazel_lint, 802 source_is_in_build_files, 803) 804 805LINTFORMAT = ( 806 _LINTFORMAT, 807 static_analysis, 808 pw_presubmit.python_checks.check_python_versions, 809 pw_presubmit.python_checks.gn_python_lint, 810) 811 812QUICK = ( 813 _LINTFORMAT, 814 gn_quick_build_check, 815 # TODO(pwbug/141): Re-enable CMake and Bazel for Mac after we have fixed the 816 # the clang issues. The problem is that all clang++ invocations need the 817 # two extra flags: "-nostdc++" and "${clang_prefix}/../lib/libc++.a". 818 cmake_clang if sys.platform != 'darwin' else (), 819) 820 821FULL = ( 822 _LINTFORMAT, 823 gn_host_build, 824 gn_arm_build, 825 gn_docs_build, 826 gn_host_tools, 827 bazel_test if sys.platform == 'linux' else (), 828 bazel_build if sys.platform == 'linux' else (), 829 # On Mac OS, system 'gcc' is a symlink to 'clang' by default, so skip GCC 830 # host builds on Mac for now. Skip it on Windows too, since gn_host_build 831 # already uses 'gcc' on Windows. 832 gn_gcc_build if sys.platform not in ('darwin', 'win32') else (), 833 # Windows doesn't support QEMU yet. 834 gn_qemu_build if sys.platform != 'win32' else (), 835 gn_qemu_clang_build if sys.platform != 'win32' else (), 836 source_is_in_build_files, 837 python_checks.gn_python_check, 838 python_checks.gn_python_test_coverage, 839 build_env_setup, 840 # Skip gn_teensy_build if running on Windows. The Teensycore installer is 841 # an exe that requires an admin role. 842 gn_teensy_build if sys.platform in ['linux', 'darwin'] else (), 843) 844 845PROGRAMS = Programs( 846 full=FULL, 847 lintformat=LINTFORMAT, 848 other_checks=OTHER_CHECKS, 849 quick=QUICK, 850) 851 852 853def parse_args() -> argparse.Namespace: 854 """Creates an argument parser and parses arguments.""" 855 856 parser = argparse.ArgumentParser(description=__doc__) 857 cli.add_arguments(parser, PROGRAMS, 'quick') 858 parser.add_argument( 859 '--install', 860 action='store_true', 861 help='Install the presubmit as a Git pre-push hook and exit.') 862 863 return parser.parse_args() 864 865 866def run(install: bool, **presubmit_args) -> int: 867 """Entry point for presubmit.""" 868 869 if install: 870 # TODO(pwbug/209, pwbug/386) inclusive-language: disable 871 install_hook(__file__, 'pre-push', 872 ['--base', 'origin/master..HEAD', '--program', 'quick'], 873 Path.cwd()) 874 # TODO(pwbug/209, pwbug/386) inclusive-language: enable 875 return 0 876 877 return cli.run(**presubmit_args) 878 879 880def main() -> int: 881 """Run the presubmit for the Pigweed repository.""" 882 return run(**vars(parse_args())) 883 884 885if __name__ == '__main__': 886 try: 887 # If pw_cli is available, use it to initialize logs. 888 from pw_cli import log 889 890 log.install(logging.INFO) 891 except ImportError: 892 # If pw_cli isn't available, display log messages like a simple print. 893 logging.basicConfig(format='%(message)s', level=logging.INFO) 894 895 sys.exit(main()) 896