1#!/usr/bin/env python3 2 3# Copyright 2020 The Pigweed Authors 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); you may not 6# use this file except in compliance with the License. You may obtain a copy of 7# the License at 8# 9# https://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14# License for the specific language governing permissions and limitations under 15# the License. 16"""Runs the local presubmit checks for the Pigweed repository.""" 17 18import argparse 19import json 20import logging 21import os 22from pathlib import Path 23import platform 24import re 25import shutil 26import subprocess 27import sys 28from typing import Callable, Iterable, Sequence, TextIO 29 30from pw_cli.plural import plural 31from pw_cli.file_filter import FileFilter 32import pw_package.pigweed_packages 33from pw_presubmit import ( 34 build, 35 cli, 36 cpp_checks, 37 format_code, 38 git_repo, 39 gitmodules, 40 inclusive_language, 41 javascript_checks, 42 json_check, 43 keep_sorted, 44 module_owners, 45 npm_presubmit, 46 owners_checks, 47 python_checks, 48 shell_checks, 49 source_in_build, 50 todo_check, 51) 52from pw_presubmit.presubmit import ( 53 Programs, 54 call, 55 filter_paths, 56) 57from pw_presubmit.presubmit_context import ( 58 FormatOptions, 59 PresubmitContext, 60 PresubmitFailure, 61) 62from pw_presubmit.tools import log_run 63from pw_presubmit.install_hook import install_git_hook 64 65_LOG = logging.getLogger(__name__) 66 67pw_package.pigweed_packages.initialize() 68 69# Trigger builds if files with these extensions change. 70_BUILD_FILE_FILTER = FileFilter( 71 suffix=( 72 *format_code.C_FORMAT.extensions, 73 '.cfg', 74 '.py', 75 '.rst', 76 '.gn', 77 '.gni', 78 '.emb', 79 ) 80) 81 82_OPTIMIZATION_LEVELS = 'debug', 'size_optimized', 'speed_optimized' 83 84 85def _at_all_optimization_levels(target): 86 for level in _OPTIMIZATION_LEVELS: 87 yield f'{target}_{level}' 88 89 90class PigweedGnGenNinja(build.GnGenNinja): 91 """Add Pigweed-specific defaults to GnGenNinja.""" 92 93 def add_default_gn_args(self, args): 94 """Add project-specific default GN args to 'args'.""" 95 args['pw_C_OPTIMIZATION_LEVELS'] = ('debug',) 96 97 98def build_bazel(*args, **kwargs) -> None: 99 build.bazel(*args, use_remote_cache=True, **kwargs) 100 101 102# 103# Build presubmit checks 104# 105gn_all = PigweedGnGenNinja( 106 name='gn_all', 107 path_filter=_BUILD_FILE_FILTER, 108 gn_args=dict(pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS), 109 ninja_targets=('all',), 110) 111 112 113def gn_clang_build(ctx: PresubmitContext): 114 """Checks all compile targets that rely on LLVM tooling.""" 115 build_targets = [ 116 *_at_all_optimization_levels('host_clang'), 117 'cpp20_compatibility', 118 'asan', 119 'tsan', 120 'ubsan', 121 'runtime_sanitizers', 122 # TODO: b/234876100 - msan will not work until the C++ standard library 123 # included in the sysroot has a variant built with msan. 124 ] 125 126 # clang-tidy doesn't run on Windows. 127 if sys.platform != 'win32': 128 build_targets.append('static_analysis') 129 130 # QEMU doesn't run on Windows. 131 if sys.platform != 'win32': 132 # TODO: b/244604080 - For the pw::InlineString tests, qemu_clang_debug 133 # and qemu_clang_speed_optimized produce a binary too large for the 134 # QEMU target's 256KB flash. Restore debug and speed optimized 135 # builds when this is fixed. 136 build_targets.append('qemu_clang_size_optimized') 137 138 # TODO: b/240982565 - SocketStream currently requires Linux. 139 if sys.platform.startswith('linux'): 140 build_targets.append('integration_tests') 141 142 build.gn_gen(ctx, pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS) 143 build.ninja(ctx, *build_targets) 144 build.gn_check(ctx) 145 146 147_HOST_COMPILER = 'gcc' if sys.platform == 'win32' else 'clang' 148 149 150@filter_paths(file_filter=_BUILD_FILE_FILTER) 151def gn_quick_build_check(ctx: PresubmitContext): 152 """Checks the state of the GN build by running gn gen and gn check.""" 153 build.gn_gen(ctx) 154 155 156def _gn_combined_build_check_targets() -> Sequence[str]: 157 build_targets = [ 158 'check_modules', 159 *_at_all_optimization_levels('stm32f429i'), 160 *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), 161 'python.tests', 162 'python.lint', 163 'pigweed_pypi_distribution', 164 ] 165 166 # TODO: b/315998985 - Add docs back to Mac ARM build. 167 if sys.platform != 'darwin' or platform.machine() != 'arm64': 168 build_targets.append('docs') 169 170 # C headers seem to be missing when building with pw_minimal_cpp_stdlib, so 171 # skip it on Windows. 172 if sys.platform != 'win32': 173 build_targets.append('build_with_pw_minimal_cpp_stdlib') 174 175 # TODO: b/234645359 - Re-enable on Windows when compatibility tests build. 176 if sys.platform != 'win32': 177 build_targets.append('cpp20_compatibility') 178 179 # clang-tidy doesn't run on Windows. 180 if sys.platform != 'win32': 181 build_targets.append('static_analysis') 182 183 # QEMU doesn't run on Windows. 184 if sys.platform != 'win32': 185 # TODO: b/244604080 - For the pw::InlineString tests, qemu_*_debug 186 # and qemu_*_speed_optimized produce a binary too large for the 187 # QEMU target's 256KB flash. Restore debug and speed optimized 188 # builds when this is fixed. 189 build_targets.append('qemu_gcc_size_optimized') 190 build_targets.append('qemu_clang_size_optimized') 191 192 # TODO: b/240982565 - SocketStream currently requires Linux. 193 if sys.platform.startswith('linux'): 194 build_targets.append('integration_tests') 195 196 # TODO: b/269354373 - clang is not supported on windows yet 197 if sys.platform != 'win32': 198 build_targets.append('host_clang_debug_dynamic_allocation') 199 200 return build_targets 201 202 203gn_combined_build_check = PigweedGnGenNinja( 204 name='gn_combined_build_check', 205 doc='Run most host and device (QEMU) tests.', 206 path_filter=_BUILD_FILE_FILTER, 207 gn_args=dict( 208 pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS, 209 pw_BUILD_BROKEN_GROUPS=True, # Enable to fully test the GN build 210 ), 211 ninja_targets=_gn_combined_build_check_targets(), 212) 213 214coverage = PigweedGnGenNinja( 215 name='coverage', 216 doc='Run coverage for the host build.', 217 path_filter=_BUILD_FILE_FILTER, 218 ninja_targets=('coverage',), 219 coverage_options=build.CoverageOptions( 220 common=build.CommonCoverageOptions( 221 target_bucket_project='pigweed', 222 target_bucket_root='gs://ng3-metrics/ng3-pigweed-coverage', 223 trace_type='LLVM', 224 owner='pigweed-infra@google.com', 225 bug_component='503634', 226 ), 227 codesearch=( 228 build.CodeSearchCoverageOptions( 229 host='pigweed-internal', 230 project='codesearch', 231 add_prefix='pigweed', 232 ref='refs/heads/main', 233 source='infra:main', 234 ), 235 ), 236 gerrit=build.GerritCoverageOptions( 237 project='pigweed/pigweed', 238 ), 239 ), 240) 241 242 243@filter_paths(file_filter=_BUILD_FILE_FILTER) 244def gn_arm_build(ctx: PresubmitContext): 245 build.gn_gen(ctx, pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS) 246 build.ninja(ctx, *_at_all_optimization_levels('stm32f429i')) 247 build.gn_check(ctx) 248 249 250stm32f429i = PigweedGnGenNinja( 251 name='stm32f429i', 252 path_filter=_BUILD_FILE_FILTER, 253 gn_args={ 254 'pw_use_test_server': True, 255 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 256 }, 257 ninja_contexts=( 258 lambda ctx: build.test_server( 259 'stm32f429i_disc1_test_server', 260 ctx.output_dir, 261 ), 262 ), 263 ninja_targets=_at_all_optimization_levels('stm32f429i'), 264) 265 266gn_crypto_mbedtls_build = PigweedGnGenNinja( 267 name='gn_crypto_mbedtls_build', 268 path_filter=_BUILD_FILE_FILTER, 269 packages=('mbedtls',), 270 gn_args={ 271 'dir_pw_third_party_mbedtls': lambda ctx: '"{}"'.format( 272 ctx.package_root / 'mbedtls' 273 ), 274 'pw_crypto_SHA256_BACKEND': lambda ctx: '"{}"'.format( 275 ctx.root / 'pw_crypto:sha256_mbedtls_v3' 276 ), 277 'pw_crypto_ECDSA_BACKEND': lambda ctx: '"{}"'.format( 278 ctx.root / 'pw_crypto:ecdsa_mbedtls_v3' 279 ), 280 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 281 }, 282 ninja_targets=( 283 *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), 284 # TODO: b/240982565 - SocketStream currently requires Linux. 285 *(('integration_tests',) if sys.platform.startswith('linux') else ()), 286 ), 287) 288 289gn_crypto_micro_ecc_build = PigweedGnGenNinja( 290 name='gn_crypto_micro_ecc_build', 291 path_filter=_BUILD_FILE_FILTER, 292 packages=('micro-ecc',), 293 gn_args={ 294 'dir_pw_third_party_micro_ecc': lambda ctx: '"{}"'.format( 295 ctx.package_root / 'micro-ecc' 296 ), 297 'pw_crypto_ECDSA_BACKEND': lambda ctx: '"{}"'.format( 298 ctx.root / 'pw_crypto:ecdsa_uecc' 299 ), 300 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 301 }, 302 ninja_targets=( 303 *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), 304 # TODO: b/240982565 - SocketStream currently requires Linux. 305 *(('integration_tests',) if sys.platform.startswith('linux') else ()), 306 ), 307) 308 309gn_teensy_build = PigweedGnGenNinja( 310 name='gn_teensy_build', 311 path_filter=_BUILD_FILE_FILTER, 312 packages=('teensy',), 313 gn_args={ 314 'pw_arduino_build_CORE_PATH': lambda ctx: '"{}"'.format( 315 str(ctx.package_root) 316 ), 317 'pw_arduino_build_CORE_NAME': 'teensy', 318 'pw_arduino_build_PACKAGE_NAME': 'avr/1.58.1', 319 'pw_arduino_build_BOARD': 'teensy40', 320 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 321 }, 322 ninja_targets=_at_all_optimization_levels('arduino'), 323) 324 325gn_pico_build = PigweedGnGenNinja( 326 name='gn_pico_build', 327 path_filter=_BUILD_FILE_FILTER, 328 packages=('pico_sdk', 'freertos'), 329 gn_args={ 330 'dir_pw_third_party_freertos': lambda ctx: '"{}"'.format( 331 str(ctx.package_root / 'freertos') 332 ), 333 'PICO_SRC_DIR': lambda ctx: '"{}"'.format( 334 str(ctx.package_root / 'pico_sdk') 335 ), 336 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 337 }, 338 ninja_targets=('pi_pico',), 339) 340 341gn_mimxrt595_build = PigweedGnGenNinja( 342 name='gn_mimxrt595_build', 343 path_filter=_BUILD_FILE_FILTER, 344 packages=('mcuxpresso',), 345 gn_args={ 346 'dir_pw_third_party_mcuxpresso': lambda ctx: '"{}"'.format( 347 str(ctx.package_root / 'mcuxpresso') 348 ), 349 'pw_target_mimxrt595_evk_MANIFEST': '$dir_pw_third_party_mcuxpresso' 350 + '/EVK-MIMXRT595_manifest_v3_13.xml', 351 'pw_third_party_mcuxpresso_SDK': '//targets/mimxrt595_evk:sample_sdk', 352 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 353 }, 354 ninja_targets=('mimxrt595'), 355) 356 357gn_mimxrt595_freertos_build = PigweedGnGenNinja( 358 name='gn_mimxrt595_freertos_build', 359 path_filter=_BUILD_FILE_FILTER, 360 packages=('freertos', 'mcuxpresso'), 361 gn_args={ 362 'dir_pw_third_party_freertos': lambda ctx: '"{}"'.format( 363 str(ctx.package_root / 'freertos') 364 ), 365 'dir_pw_third_party_mcuxpresso': lambda ctx: '"{}"'.format( 366 str(ctx.package_root / 'mcuxpresso') 367 ), 368 'pw_target_mimxrt595_evk_freertos_MANIFEST': '{}/{}'.format( 369 "$dir_pw_third_party_mcuxpresso", "EVK-MIMXRT595_manifest_v3_13.xml" 370 ), 371 'pw_third_party_mcuxpresso_SDK': '//targets/mimxrt595_evk_freertos:sdk', 372 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 373 }, 374 ninja_targets=('mimxrt595_freertos'), 375) 376 377gn_software_update_build = PigweedGnGenNinja( 378 name='gn_software_update_build', 379 path_filter=_BUILD_FILE_FILTER, 380 packages=('nanopb', 'protobuf', 'mbedtls', 'micro-ecc'), 381 gn_args={ 382 'dir_pw_third_party_protobuf': lambda ctx: '"{}"'.format( 383 ctx.package_root / 'protobuf' 384 ), 385 'dir_pw_third_party_nanopb': lambda ctx: '"{}"'.format( 386 ctx.package_root / 'nanopb' 387 ), 388 'dir_pw_third_party_micro_ecc': lambda ctx: '"{}"'.format( 389 ctx.package_root / 'micro-ecc' 390 ), 391 'pw_crypto_ECDSA_BACKEND': lambda ctx: '"{}"'.format( 392 ctx.root / 'pw_crypto:ecdsa_uecc' 393 ), 394 'dir_pw_third_party_mbedtls': lambda ctx: '"{}"'.format( 395 ctx.package_root / 'mbedtls' 396 ), 397 'pw_crypto_SHA256_BACKEND': lambda ctx: '"{}"'.format( 398 ctx.root / 'pw_crypto:sha256_mbedtls_v3' 399 ), 400 'pw_C_OPTIMIZATION_LEVELS': _OPTIMIZATION_LEVELS, 401 }, 402 ninja_targets=_at_all_optimization_levels('host_clang'), 403) 404 405gn_pw_system_demo_build = PigweedGnGenNinja( 406 name='gn_pw_system_demo_build', 407 path_filter=_BUILD_FILE_FILTER, 408 packages=('freertos', 'nanopb', 'stm32cube_f4', 'pico_sdk'), 409 gn_args={ 410 'dir_pw_third_party_freertos': lambda ctx: '"{}"'.format( 411 ctx.package_root / 'freertos' 412 ), 413 'dir_pw_third_party_nanopb': lambda ctx: '"{}"'.format( 414 ctx.package_root / 'nanopb' 415 ), 416 'dir_pw_third_party_stm32cube_f4': lambda ctx: '"{}"'.format( 417 ctx.package_root / 'stm32cube_f4' 418 ), 419 'PICO_SRC_DIR': lambda ctx: '"{}"'.format( 420 str(ctx.package_root / 'pico_sdk') 421 ), 422 }, 423 ninja_targets=('pw_system_demo',), 424) 425 426gn_chre_googletest_nanopb_sapphire_build = PigweedGnGenNinja( 427 name='gn_chre_googletest_nanopb_sapphire_build', 428 path_filter=_BUILD_FILE_FILTER, 429 packages=('boringssl', 'chre', 'emboss', 'googletest', 'icu', 'nanopb'), 430 gn_args=dict( 431 dir_pw_third_party_chre=lambda ctx: '"{}"'.format( 432 ctx.package_root / 'chre' 433 ), 434 dir_pw_third_party_nanopb=lambda ctx: '"{}"'.format( 435 ctx.package_root / 'nanopb' 436 ), 437 dir_pw_third_party_googletest=lambda ctx: '"{}"'.format( 438 ctx.package_root / 'googletest' 439 ), 440 dir_pw_third_party_emboss=lambda ctx: '"{}"'.format( 441 ctx.package_root / 'emboss' 442 ), 443 dir_pw_third_party_boringssl=lambda ctx: '"{}"'.format( 444 ctx.package_root / 'boringssl' 445 ), 446 dir_pw_third_party_icu=lambda ctx: '"{}"'.format( 447 ctx.package_root / 'icu' 448 ), 449 pw_unit_test_MAIN=lambda ctx: '"{}"'.format( 450 ctx.root / 'third_party/googletest:gmock_main' 451 ), 452 pw_unit_test_BACKEND=lambda ctx: '"{}"'.format( 453 ctx.root / 'pw_unit_test:googletest' 454 ), 455 pw_function_CONFIG=lambda ctx: '"{}"'.format( 456 ctx.root / 'pw_function:enable_dynamic_allocation' 457 ), 458 pw_bluetooth_sapphire_ENABLED=True, 459 pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS, 460 ), 461 ninja_targets=( 462 *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), 463 *_at_all_optimization_levels('stm32f429i'), 464 ), 465) 466 467gn_fuzz_build = PigweedGnGenNinja( 468 name='gn_fuzz_build', 469 path_filter=_BUILD_FILE_FILTER, 470 packages=('abseil-cpp', 'fuzztest', 'googletest', 're2'), 471 gn_args={ 472 'dir_pw_third_party_abseil_cpp': lambda ctx: '"{}"'.format( 473 ctx.package_root / 'abseil-cpp' 474 ), 475 'dir_pw_third_party_fuzztest': lambda ctx: '"{}"'.format( 476 ctx.package_root / 'fuzztest' 477 ), 478 'dir_pw_third_party_googletest': lambda ctx: '"{}"'.format( 479 ctx.package_root / 'googletest' 480 ), 481 'dir_pw_third_party_re2': lambda ctx: '"{}"'.format( 482 ctx.package_root / 're2' 483 ), 484 'pw_unit_test_MAIN': lambda ctx: '"{}"'.format( 485 ctx.root / 'third_party/googletest:gmock_main' 486 ), 487 'pw_unit_test_BACKEND': lambda ctx: '"{}"'.format( 488 ctx.root / 'pw_unit_test:googletest' 489 ), 490 }, 491 ninja_targets=('fuzzers',), 492 ninja_contexts=( 493 lambda ctx: build.modified_env( 494 FUZZTEST_PRNG_SEED=build.fuzztest_prng_seed(ctx), 495 ), 496 ), 497) 498 499oss_fuzz_build = PigweedGnGenNinja( 500 name='oss_fuzz_build', 501 path_filter=_BUILD_FILE_FILTER, 502 packages=('abseil-cpp', 'fuzztest', 'googletest', 're2'), 503 gn_args={ 504 'dir_pw_third_party_abseil_cpp': lambda ctx: '"{}"'.format( 505 ctx.package_root / 'abseil-cpp' 506 ), 507 'dir_pw_third_party_fuzztest': lambda ctx: '"{}"'.format( 508 ctx.package_root / 'fuzztest' 509 ), 510 'dir_pw_third_party_googletest': lambda ctx: '"{}"'.format( 511 ctx.package_root / 'googletest' 512 ), 513 'dir_pw_third_party_re2': lambda ctx: '"{}"'.format( 514 ctx.package_root / 're2' 515 ), 516 'pw_toolchain_OSS_FUZZ_ENABLED': True, 517 }, 518 ninja_targets=('oss_fuzz',), 519) 520 521 522def _env_with_zephyr_vars(ctx: PresubmitContext) -> dict: 523 """Returns the environment variables with ... set for Zephyr.""" 524 env = os.environ.copy() 525 # Set some variables here. 526 env['ZEPHYR_BASE'] = str(ctx.package_root / 'zephyr') 527 env['ZEPHYR_MODULES'] = str(ctx.root) 528 env['ZEPHYR_TOOLCHAIN_VARIANT'] = 'llvm' 529 return env 530 531 532def zephyr_build(ctx: PresubmitContext) -> None: 533 """Run Zephyr compatible tests""" 534 # Install the Zephyr package 535 build.install_package(ctx, 'zephyr') 536 # Configure the environment 537 env = _env_with_zephyr_vars(ctx) 538 # Get the python twister runner 539 twister = ctx.package_root / 'zephyr' / 'scripts' / 'twister' 540 # Get a list of the test roots 541 testsuite_roots = [ 542 ctx.pw_root / dir 543 for dir in os.listdir(ctx.pw_root) 544 if dir.startswith('pw_') 545 ] 546 testsuite_roots_list = [ 547 args for dir in testsuite_roots for args in ('--testsuite-root', dir) 548 ] 549 sysroot_dir = ( 550 ctx.pw_root 551 / 'environment' 552 / 'cipd' 553 / 'packages' 554 / 'pigweed' 555 / 'clang_sysroot' 556 ) 557 platform_filters = ( 558 ['-P', 'native_posix', '-P', 'native_sim'] 559 if platform.system() in ['Windows', 'Darwin'] 560 else [] 561 ) 562 # Run twister 563 call( 564 sys.executable, 565 twister, 566 '--ninja', 567 '--integration', 568 '--clobber-output', 569 '--inline-logs', 570 '--verbose', 571 *platform_filters, 572 '-x=CONFIG_LLVM_USE_LLD=y', 573 '-x=CONFIG_COMPILER_RT_RTLIB=y', 574 f'-x=TOOLCHAIN_C_FLAGS=--sysroot={sysroot_dir}', 575 f'-x=TOOLCHAIN_LD_FLAGS=--sysroot={sysroot_dir}', 576 *testsuite_roots_list, 577 env=env, 578 ) 579 # Produces reports at (ctx.root / 'twister_out' / 'twister*.xml') 580 581 582def docs_build(ctx: PresubmitContext) -> None: 583 """Build Pigweed docs""" 584 585 build.install_package(ctx, 'nanopb') 586 build.install_package(ctx, 'pico_sdk') 587 build.install_package(ctx, 'stm32cube_f4') 588 build.install_package(ctx, 'freertos') 589 build.install_package(ctx, 'pigweed_examples_repo') 590 591 # Build main docs through GN/Ninja. 592 build.gn_gen(ctx, pw_C_OPTIMIZATION_LEVELS=_OPTIMIZATION_LEVELS) 593 build.ninja(ctx, 'docs') 594 build.gn_check(ctx) 595 596 # Build Rust docs through Bazel. 597 build_bazel( 598 ctx, 599 'build', 600 '--', 601 '//pw_rust:docs', 602 ) 603 604 # Build examples repo docs through GN. 605 examples_repo_root = ctx.package_root / 'pigweed_examples_repo' 606 examples_repo_out = examples_repo_root / 'out' 607 608 # Setup an examples repo presubmit context. 609 examples_ctx = PresubmitContext( 610 root=examples_repo_root, 611 repos=(examples_repo_root,), 612 output_dir=examples_repo_out, 613 failure_summary_log=ctx.failure_summary_log, 614 paths=tuple(), 615 all_paths=tuple(), 616 package_root=ctx.package_root, 617 luci=None, 618 override_gn_args={}, 619 num_jobs=ctx.num_jobs, 620 continue_after_build_error=True, 621 _failed=False, 622 format_options=ctx.format_options, 623 ) 624 625 # Write a pigweed_environment.gni for the examples repo. 626 pwenvgni = ( 627 ctx.root / 'build_overrides/pigweed_environment.gni' 628 ).read_text() 629 # Fix the path for cipd packages. 630 pwenvgni.replace('../environment/cipd/', '../../cipd/') 631 # Write the file 632 (examples_repo_root / 'build_overrides/pigweed_environment.gni').write_text( 633 pwenvgni 634 ) 635 636 # Set required GN args. 637 pico_sdk_dir = ctx.package_root / 'pico_sdk' 638 stm32cube_dir = ctx.package_root / 'stm32cube_f4' 639 freertos_dir = ctx.package_root / 'freertos' 640 nanopb_dir = ctx.package_root / 'nanopb' 641 build.gn_gen( 642 examples_ctx, 643 dir_pigweed='"//../../.."', 644 dir_pw_third_party_stm32cube_f4=f'"{stm32cube_dir}"', 645 dir_pw_third_party_freertos=f'"{freertos_dir}"', 646 dir_pw_third_party_nanopb=f'"{nanopb_dir}"', 647 PICO_SRC_DIR=f'"{pico_sdk_dir}"', 648 ) 649 build.ninja(examples_ctx, 'docs') 650 651 # Copy rust docs from Bazel's out directory into where the GN build 652 # put the main docs. 653 rust_docs_bazel_dir = ctx.output_dir / 'bazel-bin/pw_rust/docs.rustdoc' 654 rust_docs_output_dir = ctx.output_dir / 'docs/gen/docs/html/rustdoc' 655 656 # Copy the doxygen html output to the main docs location. 657 doxygen_html_gn_dir = ctx.output_dir / 'docs/doxygen/html' 658 doxygen_html_output_dir = ctx.output_dir / 'docs/gen/docs/html/doxygen' 659 660 # Copy the examples repo html output to the main docs location into 661 # '/examples/'. 662 examples_html_gn_dir = examples_repo_out / 'docs/gen/docs/html' 663 examples_html_output_dir = ctx.output_dir / 'docs/gen/docs/html/examples' 664 665 # Remove outputs to avoid including stale files from previous runs. 666 shutil.rmtree(rust_docs_output_dir, ignore_errors=True) 667 shutil.rmtree(doxygen_html_output_dir, ignore_errors=True) 668 shutil.rmtree(examples_html_output_dir, ignore_errors=True) 669 670 # Bazel generates files and directories without write permissions. In 671 # order to allow this rule to be run multiple times we use shutil.copyfile 672 # for the actual copies to not copy permissions of files. 673 shutil.copytree( 674 rust_docs_bazel_dir, 675 rust_docs_output_dir, 676 copy_function=shutil.copyfile, 677 dirs_exist_ok=True, 678 ) 679 680 # Copy doxygen html outputs. 681 shutil.copytree( 682 doxygen_html_gn_dir, 683 doxygen_html_output_dir, 684 copy_function=shutil.copyfile, 685 dirs_exist_ok=True, 686 ) 687 688 # mkdir -p the example repo output dir and copy the files over. 689 examples_html_output_dir.mkdir(parents=True, exist_ok=True) 690 shutil.copytree( 691 examples_html_gn_dir, 692 examples_html_output_dir, 693 copy_function=shutil.copyfile, 694 dirs_exist_ok=True, 695 ) 696 697 698gn_host_tools = PigweedGnGenNinja( 699 name='gn_host_tools', 700 ninja_targets=('host_tools',), 701) 702 703 704def _run_cmake(ctx: PresubmitContext, toolchain='host_clang') -> None: 705 build.install_package(ctx, 'nanopb') 706 build.install_package(ctx, 'emboss') 707 708 env = None 709 if 'clang' in toolchain: 710 env = build.env_with_clang_vars() 711 712 toolchain_path = ctx.root / 'pw_toolchain' / toolchain / 'toolchain.cmake' 713 build.cmake( 714 ctx, 715 f'-DCMAKE_TOOLCHAIN_FILE={toolchain_path}', 716 '-DCMAKE_EXPORT_COMPILE_COMMANDS=1', 717 f'-Ddir_pw_third_party_nanopb={ctx.package_root / "nanopb"}', 718 '-Dpw_third_party_nanopb_ADD_SUBDIRECTORY=ON', 719 f'-Ddir_pw_third_party_emboss={ctx.package_root / "emboss"}', 720 env=env, 721 ) 722 723 724CMAKE_TARGETS = [ 725 'pw_apps', 726 'pw_run_tests.modules', 727] 728 729 730@filter_paths( 731 endswith=(*format_code.C_FORMAT.extensions, '.cmake', 'CMakeLists.txt') 732) 733def cmake_clang(ctx: PresubmitContext): 734 _run_cmake(ctx, toolchain='host_clang') 735 build.ninja(ctx, *CMAKE_TARGETS) 736 build.gn_check(ctx) 737 738 739@filter_paths( 740 endswith=(*format_code.C_FORMAT.extensions, '.cmake', 'CMakeLists.txt') 741) 742def cmake_gcc(ctx: PresubmitContext): 743 _run_cmake(ctx, toolchain='host_gcc') 744 build.ninja(ctx, *CMAKE_TARGETS) 745 build.gn_check(ctx) 746 747 748@filter_paths( 749 endswith=(*format_code.C_FORMAT.extensions, '.bazel', '.bzl', 'BUILD') 750) 751def bazel_test(ctx: PresubmitContext) -> None: 752 """Runs bazel test on the entire repo.""" 753 build_bazel( 754 ctx, 755 'test', 756 '--build_tag_filters=-requires_cxx_20', 757 '--test_tag_filters=-requires_cxx_20', 758 '--', 759 '//...', 760 ) 761 762 # Run tests for non-default config options 763 764 # pw_rpc 765 build_bazel( 766 ctx, 767 'test', 768 '--//pw_rpc:config_override=' 769 '//pw_rpc:completion_request_callback_config_enabled', 770 '--', 771 '//pw_rpc/...', 772 ) 773 774 # pw_grpc 775 build_bazel( 776 ctx, 777 'test', 778 '--//pw_rpc:config_override=//pw_grpc:pw_rpc_config', 779 '--', 780 '//pw_grpc/...', 781 ) 782 783 784def bthost_package(ctx: PresubmitContext) -> None: 785 target = '//pw_bluetooth_sapphire/fuchsia:infra' 786 build_bazel(ctx, 'build', target) 787 build_bazel(ctx, 'test', f'{target}.test_all') 788 789 stdout_path = ctx.output_dir / 'bazel.manifest.stdout' 790 with open(stdout_path, 'w') as outs: 791 build_bazel( 792 ctx, 793 'build', 794 '--output_groups=builder_manifest', 795 target, 796 stdout=outs, 797 ) 798 799 manifest_path: Path | None = None 800 for line in stdout_path.read_text().splitlines(): 801 line = line.strip() 802 if line.endswith('infrabuilder_manifest.json'): 803 manifest_path = Path(line) 804 break 805 else: 806 raise PresubmitFailure('no manifest found in output') 807 808 _LOG.debug('manifest: %s', manifest_path) 809 shutil.copyfile(manifest_path, ctx.output_dir / 'builder_manifest.json') 810 811 812@filter_paths( 813 endswith=( 814 *format_code.C_FORMAT.extensions, 815 '.bazel', 816 '.bzl', 817 '.py', 818 '.rs', 819 'BUILD', 820 ) 821) 822def bazel_build(ctx: PresubmitContext) -> None: 823 """Runs Bazel build for each supported platform.""" 824 # Build everything with the default flags. 825 build_bazel( 826 ctx, 827 'build', 828 '--build_tag_filters=-requires_cxx_20', 829 '--', 830 '//...', 831 ) 832 833 # Mapping from Bazel platforms to targets which should be built for those 834 # platforms. 835 targets_for_config = { 836 "lm3s6965evb": [ 837 "//pw_rust/...", 838 ], 839 "microbit": [ 840 "//pw_rust/...", 841 ], 842 } 843 844 for cxxversion in ('c++17', 'c++20'): 845 # Explicitly build for each supported C++ version. 846 args = [ctx, 'build', f"--cxxopt=-std={cxxversion}"] 847 if cxxversion == 'c++17': 848 args += ['--build_tag_filters=-requires_cxx_20'] 849 args += ['--', '//...'] 850 build_bazel(*args) 851 852 for config, targets in targets_for_config.items(): 853 build_bazel( 854 ctx, 855 'build', 856 f'--config={config}', 857 f"--cxxopt='-std={cxxversion}'", 858 *targets, 859 ) 860 861 # Provide some coverage of the FreeRTOS build. 862 # 863 # This is just a minimal presubmit intended to ensure we don't break what 864 # support we have. 865 # 866 # TODO: b/271465588 - Eventually just build the entire repo for this 867 # platform. 868 build_bazel( 869 ctx, 870 'build', 871 '--platforms=//pw_build/platforms:testonly_freertos', 872 '//pw_sync/...', 873 '//pw_thread/...', 874 '//pw_thread_freertos/...', 875 '//pw_interrupt/...', 876 '//pw_cpu_exception/...', 877 ) 878 879 build_bazel( 880 ctx, 881 'build', 882 '--//pw_thread_freertos:config_override=//pw_build:test_module_config', 883 '--platforms=//pw_build/platforms:testonly_freertos', 884 '//pw_build:module_config_test', 885 ) 886 887 # Provide some coverage of the RP2040 build. 888 # 889 # This is just a minimal presubmit intended to ensure we don't break what 890 # support we have. 891 # 892 # TODO: b/271465588 - Eventually just build the entire repo for this 893 # platform. 894 build_bazel( 895 ctx, 896 'build', 897 '--config=rp2040', 898 '//pw_system:system_example', 899 ) 900 901 # Build the pw_system example for the Discovery board using STM32Cube. 902 build_bazel( 903 ctx, 904 'build', 905 '--config=stm32f429i', 906 '//pw_system:system_example', 907 ) 908 909 # Build the fuzztest example. 910 # 911 # TODO: b/324652164 - This doesn't work on MacOS yet. 912 if sys.platform != 'darwin': 913 build_bazel( 914 ctx, 915 'build', 916 '--config=fuzztest', 917 '//pw_fuzzer/examples/fuzztest:metrics_fuzztest', 918 ) 919 920 921def pw_transfer_integration_test(ctx: PresubmitContext) -> None: 922 """Runs the pw_transfer cross-language integration test only. 923 924 This test is not part of the regular bazel build because it's slow and 925 intended to run in CI only. 926 """ 927 build_bazel( 928 ctx, 929 'test', 930 '//pw_transfer/integration_test:cross_language_small_test', 931 '//pw_transfer/integration_test:cross_language_medium_read_test', 932 '//pw_transfer/integration_test:cross_language_medium_write_test', 933 '//pw_transfer/integration_test:cross_language_large_read_test', 934 '//pw_transfer/integration_test:cross_language_large_write_test', 935 '//pw_transfer/integration_test:multi_transfer_test', 936 '//pw_transfer/integration_test:expected_errors_test', 937 '//pw_transfer/integration_test:legacy_binaries_test', 938 '--test_output=errors', 939 ) 940 941 942# 943# General presubmit checks 944# 945 946 947def _clang_system_include_paths(lang: str) -> list[str]: 948 """Generate default system header paths. 949 950 Returns the list of system include paths used by the host 951 clang installation. 952 """ 953 # Dump system include paths with preprocessor verbose. 954 command = [ 955 'clang++', 956 '-Xpreprocessor', 957 '-v', 958 '-x', 959 f'{lang}', 960 f'{os.devnull}', 961 '-fsyntax-only', 962 ] 963 process = log_run( 964 command, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT 965 ) 966 967 # Parse the command output to retrieve system include paths. 968 # The paths are listed one per line. 969 output = process.stdout.decode(errors='backslashreplace') 970 include_paths: list[str] = [] 971 for line in output.splitlines(): 972 path = line.strip() 973 if os.path.exists(path): 974 include_paths.append(f'-isystem{path}') 975 976 return include_paths 977 978 979def edit_compile_commands( 980 in_path: Path, out_path: Path, func: Callable[[str, str, str], str] 981) -> None: 982 """Edit the selected compile command file. 983 984 Calls the input callback on all triplets (file, directory, command) in 985 the input compile commands database. The return value replaces the old 986 compile command in the output database. 987 """ 988 with open(in_path) as in_file: 989 compile_commands = json.load(in_file) 990 for item in compile_commands: 991 item['command'] = func( 992 item['file'], item['directory'], item['command'] 993 ) 994 with open(out_path, 'w') as out_file: 995 json.dump(compile_commands, out_file, indent=2) 996 997 998_EXCLUDE_FROM_COPYRIGHT_NOTICE: Sequence[str] = ( 999 # Configuration 1000 # keep-sorted: start 1001 r'MODULE.bazel.lock', 1002 r'\bDoxyfile$', 1003 r'\bPW_PLUGINS$', 1004 r'\bconstraint.list$', 1005 r'\bconstraint_hashes_darwin.list$', 1006 r'\bconstraint_hashes_linux.list$', 1007 r'\bconstraint_hashes_windows.list$', 1008 r'\bpython_base_requirements.txt$', 1009 r'\bupstream_requirements_darwin_lock.txt$', 1010 r'\bupstream_requirements_linux_lock.txt$', 1011 r'\bupstream_requirements_windows_lock.txt$', 1012 r'^(?:.+/)?\..+$', 1013 # keep-sorted: end 1014 # Metadata 1015 # keep-sorted: start 1016 r'\b.*OWNERS.*$', 1017 r'\bAUTHORS$', 1018 r'\bLICENSE$', 1019 r'\bPIGWEED_MODULES$', 1020 r'\bgo.(mod|sum)$', 1021 r'\bpackage-lock.json$', 1022 r'\bpackage.json$', 1023 r'\brequirements.txt$', 1024 r'\byarn.lock$', 1025 r'^docker/tag$', 1026 r'^patches.json$', 1027 # keep-sorted: end 1028 # Data files 1029 # keep-sorted: start 1030 r'\.bin$', 1031 r'\.csv$', 1032 r'\.elf$', 1033 r'\.gif$', 1034 r'\.ico$', 1035 r'\.jpg$', 1036 r'\.json$', 1037 r'\.png$', 1038 r'\.svg$', 1039 r'\.vsix$', 1040 r'\.xml$', 1041 # keep-sorted: end 1042 # Documentation 1043 # keep-sorted: start 1044 r'\.md$', 1045 r'\.rst$', 1046 # keep-sorted: end 1047 # Generated protobuf files 1048 # keep-sorted: start 1049 r'\.pb\.c$', 1050 r'\.pb\.h$', 1051 r'\_pb2.pyi?$', 1052 # keep-sorted: end 1053 # Generated third-party files 1054 # keep-sorted: start 1055 r'\bthird_party/.*\.bazelrc$', 1056 r'\bthird_party/perfetto/repo/protos/perfetto/trace/perfetto_trace.proto', 1057 # keep-sorted: end 1058 # Diff/Patch files 1059 # keep-sorted: start 1060 r'\.diff$', 1061 r'\.patch$', 1062 # keep-sorted: end 1063 # Test data 1064 # keep-sorted: start 1065 r'\bpw_presubmit/py/test/owners_checks/', 1066 # keep-sorted: end 1067) 1068 1069# Regular expression for the copyright comment. "\1" refers to the comment 1070# characters and "\2" refers to space after the comment characters, if any. 1071# All period characters are escaped using a replace call. 1072# pylint: disable=line-too-long 1073_COPYRIGHT = re.compile( 1074 r"""(#|//|::| \*|)( ?)Copyright 2\d{3} The Pigweed Authors 1075\1 1076\1\2Licensed under the Apache License, Version 2.0 \(the "License"\); you may not 1077\1\2use this file except in compliance with the License. You may obtain a copy of 1078\1\2the License at 1079\1 1080\1(?:\2 |\t)https://www.apache.org/licenses/LICENSE-2.0 1081\1 1082\1\2Unless required by applicable law or agreed to in writing, software 1083\1\2distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 1084\1\2WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 1085\1\2License for the specific language governing permissions and limitations under 1086\1\2the License. 1087""".replace( 1088 '.', r'\.' 1089 ), 1090 re.MULTILINE, 1091) 1092# pylint: enable=line-too-long 1093 1094_SKIP_LINE_PREFIXES = ( 1095 '#!', 1096 '#autoload', 1097 '#compdef', 1098 '@echo off', 1099 ':<<', 1100 '/*', 1101 ' * @jest-environment jsdom', 1102 ' */', 1103 '{#', # Jinja comment block 1104 '# -*- coding: utf-8 -*-', 1105 '<!--', 1106) 1107 1108 1109def _read_notice_lines(file: TextIO) -> Iterable[str]: 1110 lines = iter(file) 1111 try: 1112 # Read until the first line of the copyright notice. 1113 line = next(lines) 1114 while line.isspace() or line.startswith(_SKIP_LINE_PREFIXES): 1115 line = next(lines) 1116 1117 yield line 1118 1119 for _ in range(12): # The notice is 13 lines; read the remaining 12. 1120 yield next(lines) 1121 except StopIteration: 1122 return 1123 1124 1125@filter_paths(exclude=_EXCLUDE_FROM_COPYRIGHT_NOTICE) 1126def copyright_notice(ctx: PresubmitContext): 1127 """Checks that the Pigweed copyright notice is present.""" 1128 errors = [] 1129 1130 for path in ctx.paths: 1131 if path.stat().st_size == 0: 1132 continue # Skip empty files 1133 1134 try: 1135 with path.open() as file: 1136 if not _COPYRIGHT.match(''.join(_read_notice_lines(file))): 1137 errors.append(path) 1138 except UnicodeDecodeError as exc: 1139 raise PresubmitFailure(f'failed to read {path}') from exc 1140 1141 if errors: 1142 _LOG.warning( 1143 '%s with a missing or incorrect copyright notice:\n%s', 1144 plural(errors, 'file'), 1145 '\n'.join(str(e) for e in errors), 1146 ) 1147 raise PresubmitFailure 1148 1149 1150@filter_paths(endswith=format_code.CPP_SOURCE_EXTS) 1151def source_is_in_cmake_build_warn_only(ctx: PresubmitContext): 1152 """Checks that source files are in the CMake build.""" 1153 1154 _run_cmake(ctx) 1155 missing = SOURCE_FILES_FILTER_CMAKE_EXCLUDE.filter( 1156 build.check_compile_commands_for_files( 1157 ctx.output_dir / 'compile_commands.json', 1158 (f for f in ctx.paths if f.suffix in format_code.CPP_SOURCE_EXTS), 1159 ) 1160 ) 1161 if missing: 1162 _LOG.warning( 1163 'Files missing from CMake:\n%s', 1164 '\n'.join(str(f) for f in missing), 1165 ) 1166 1167 1168def build_env_setup(ctx: PresubmitContext): 1169 if 'PW_CARGO_SETUP' not in os.environ: 1170 _LOG.warning('Skipping build_env_setup since PW_CARGO_SETUP is not set') 1171 return 1172 1173 tmpl = ctx.root.joinpath('pw_env_setup', 'py', 'pyoxidizer.bzl.tmpl') 1174 out = ctx.output_dir.joinpath('pyoxidizer.bzl') 1175 1176 with open(tmpl, 'r') as ins: 1177 cfg = ins.read().replace('${PW_ROOT}', str(ctx.root)) 1178 with open(out, 'w') as outs: 1179 outs.write(cfg) 1180 1181 call('pyoxidizer', 'build', cwd=ctx.output_dir) 1182 1183 1184def _valid_capitalization(word: str) -> bool: 1185 """Checks that the word has a capital letter or is not a regular word.""" 1186 return bool( 1187 any(c.isupper() for c in word) # Any capitalizatian (iTelephone) 1188 or not word.isalpha() # Non-alphabetical (cool_stuff.exe) 1189 or shutil.which(word) 1190 ) # Matches an executable (clangd) 1191 1192 1193def commit_message_format(_: PresubmitContext): 1194 """Checks that the top commit's message is correctly formatted.""" 1195 if git_repo.commit_author().endswith('gserviceaccount.com'): 1196 return 1197 1198 lines = git_repo.commit_message().splitlines() 1199 1200 # Show limits and current commit message in log. 1201 _LOG.debug('%-25s%+25s%+22s', 'Line limits', '72|', '72|') 1202 for line in lines: 1203 _LOG.debug(line) 1204 1205 if not lines: 1206 _LOG.error('The commit message is too short!') 1207 raise PresubmitFailure 1208 1209 # Ignore Gerrit-generated reverts. 1210 if ( 1211 'Revert' in lines[0] 1212 and 'This reverts commit ' in git_repo.commit_message() 1213 and 'Reason for revert: ' in git_repo.commit_message() 1214 ): 1215 _LOG.warning('Ignoring apparent Gerrit-generated revert') 1216 return 1217 1218 # Ignore Gerrit-generated relands 1219 if ( 1220 'Reland' in lines[0] 1221 and 'This is a reland of ' in git_repo.commit_message() 1222 and "Original change's description:" in git_repo.commit_message() 1223 ): 1224 _LOG.warning('Ignoring apparent Gerrit-generated reland') 1225 return 1226 1227 errors = 0 1228 1229 if len(lines[0]) > 72: 1230 _LOG.warning( 1231 "The commit message's first line must be no longer than " 1232 '72 characters.' 1233 ) 1234 _LOG.warning( 1235 'The first line is %d characters:\n %s', len(lines[0]), lines[0] 1236 ) 1237 errors += 1 1238 1239 if lines[0].endswith('.'): 1240 _LOG.warning( 1241 "The commit message's first line must not end with a period:\n %s", 1242 lines[0], 1243 ) 1244 errors += 1 1245 1246 # Check that the first line matches the expected pattern. 1247 match = re.match( 1248 r'^(?:[.\w*/]+(?:{[\w* ,]+})?[\w*/]*|SEED-\d+): (?P<desc>.+)$', lines[0] 1249 ) 1250 if not match: 1251 _LOG.warning('The first line does not match the expected format') 1252 _LOG.warning( 1253 'Expected:\n\n module_or_target: The description\n\n' 1254 'Found:\n\n %s\n', 1255 lines[0], 1256 ) 1257 errors += 1 1258 elif not _valid_capitalization(match.group('desc').split()[0]): 1259 _LOG.warning( 1260 'The first word after the ":" in the first line ("%s") must be ' 1261 'capitalized:\n %s', 1262 match.group('desc').split()[0], 1263 lines[0], 1264 ) 1265 errors += 1 1266 1267 if len(lines) > 1 and lines[1]: 1268 _LOG.warning("The commit message's second line must be blank.") 1269 _LOG.warning( 1270 'The second line has %d characters:\n %s', len(lines[1]), lines[1] 1271 ) 1272 errors += 1 1273 1274 # Ignore the line length check for Copybara imports so they can include the 1275 # commit hash and description for imported commits. 1276 if not errors and ( 1277 'Copybara import' in lines[0] 1278 and 'GitOrigin-RevId:' in git_repo.commit_message() 1279 ): 1280 _LOG.warning('Ignoring Copybara import') 1281 return 1282 1283 # Check that the lines are 72 characters or less. 1284 for i, line in enumerate(lines[2:], 3): 1285 # Skip any lines that might possibly have a URL, path, or metadata in 1286 # them. 1287 if any(c in line for c in ':/>'): 1288 continue 1289 1290 # Skip any lines with non-ASCII characters. 1291 if not line.isascii(): 1292 continue 1293 1294 # Skip any blockquoted lines. 1295 if line.startswith(' '): 1296 continue 1297 1298 if len(line) > 72: 1299 _LOG.warning( 1300 'Commit message lines must be no longer than 72 characters.' 1301 ) 1302 _LOG.warning('Line %d has %d characters:\n %s', i, len(line), line) 1303 errors += 1 1304 1305 if errors: 1306 _LOG.error('Found %s in the commit message', plural(errors, 'error')) 1307 raise PresubmitFailure 1308 1309 1310@filter_paths(endswith=(*format_code.C_FORMAT.extensions, '.py')) 1311def static_analysis(ctx: PresubmitContext): 1312 """Runs all available static analysis tools.""" 1313 build.gn_gen(ctx) 1314 build.ninja(ctx, 'python.lint', 'static_analysis') 1315 build.gn_check(ctx) 1316 1317 1318_EXCLUDE_FROM_TODO_CHECK = ( 1319 # keep-sorted: start 1320 r'.bazelrc$', 1321 r'.dockerignore$', 1322 r'.gitignore$', 1323 r'.pylintrc$', 1324 r'\bdocs/build_system.rst', 1325 r'\bdocs/code_reviews.rst', 1326 r'\bpw_assert_basic/basic_handler.cc', 1327 r'\bpw_assert_basic/public/pw_assert_basic/handler.h', 1328 r'\bpw_blob_store/public/pw_blob_store/flat_file_system_entry.h', 1329 r'\bpw_build/linker_script.gni', 1330 r'\bpw_build/py/pw_build/copy_from_cipd.py', 1331 r'\bpw_cpu_exception/basic_handler.cc', 1332 r'\bpw_cpu_exception_cortex_m/entry.cc', 1333 r'\bpw_cpu_exception_cortex_m/exception_entry_test.cc', 1334 r'\bpw_doctor/py/pw_doctor/doctor.py', 1335 r'\bpw_env_setup/util.sh', 1336 r'\bpw_fuzzer/fuzzer.gni', 1337 r'\bpw_i2c/BUILD.gn', 1338 r'\bpw_i2c/public/pw_i2c/register_device.h', 1339 r'\bpw_kvs/flash_memory.cc', 1340 r'\bpw_kvs/key_value_store.cc', 1341 r'\bpw_log_basic/log_basic.cc', 1342 r'\bpw_package/py/pw_package/packages/chromium_verifier.py', 1343 r'\bpw_protobuf/encoder.cc', 1344 r'\bpw_rpc/docs.rst', 1345 r'\bpw_watch/py/pw_watch/watch.py', 1346 r'\btargets/mimxrt595_evk/BUILD.bazel', 1347 r'\btargets/stm32f429i_disc1/boot.cc', 1348 r'\bthird_party/chromium_verifier/BUILD.gn', 1349 # keep-sorted: end 1350) 1351 1352 1353@filter_paths(exclude=_EXCLUDE_FROM_TODO_CHECK) 1354def todo_check_with_exceptions(ctx: PresubmitContext): 1355 """Check that non-legacy TODO lines are valid.""" # todo-check: ignore 1356 todo_check.create(todo_check.BUGS_OR_USERNAMES)(ctx) 1357 1358 1359@filter_paths(file_filter=format_code.OWNERS_CODE_FORMAT.filter) 1360def owners_lint_checks(ctx: PresubmitContext): 1361 """Runs OWNERS linter.""" 1362 owners_checks.presubmit_check(ctx.paths) 1363 1364 1365SOURCE_FILES_FILTER = FileFilter( 1366 endswith=_BUILD_FILE_FILTER.endswith, 1367 suffix=('.bazel', '.bzl', '.gn', '.gni', *_BUILD_FILE_FILTER.suffix), 1368 exclude=( 1369 r'zephyr.*', 1370 r'android.*', 1371 r'\.black.toml', 1372 r'pyproject.toml', 1373 ), 1374) 1375 1376SOURCE_FILES_FILTER_GN_EXCLUDE = FileFilter( 1377 exclude=( 1378 # keep-sorted: start 1379 r'\bpw_bluetooth_sapphire/fuchsia', 1380 # keep-sorted: end 1381 ), 1382) 1383 1384SOURCE_FILES_FILTER_CMAKE_EXCLUDE = FileFilter( 1385 exclude=( 1386 # keep-sorted: start 1387 r'\bpw_bluetooth_sapphire/fuchsia', 1388 # keep-sorted: end 1389 ), 1390) 1391 1392# 1393# Presubmit check programs 1394# 1395 1396OTHER_CHECKS = ( 1397 # keep-sorted: start 1398 bazel_test, 1399 bthost_package, 1400 build.gn_gen_check, 1401 cmake_clang, 1402 cmake_gcc, 1403 coverage, 1404 # TODO: b/234876100 - Remove once msan is added to all_sanitizers(). 1405 cpp_checks.msan, 1406 docs_build, 1407 gitmodules.create(gitmodules.Config(allow_submodules=False)), 1408 gn_all, 1409 gn_clang_build, 1410 gn_combined_build_check, 1411 module_owners.presubmit_check(), 1412 npm_presubmit.npm_test, 1413 pw_transfer_integration_test, 1414 python_checks.update_upstream_python_constraints, 1415 python_checks.vendor_python_wheels, 1416 shell_checks.shellcheck, 1417 # TODO(hepler): Many files are missing from the CMake build. Add this check 1418 # to lintformat when the missing files are fixed. 1419 source_in_build.cmake(SOURCE_FILES_FILTER, _run_cmake), 1420 static_analysis, 1421 stm32f429i, 1422 todo_check.create(todo_check.BUGS_OR_USERNAMES), 1423 zephyr_build, 1424 # keep-sorted: end 1425) 1426 1427ARDUINO_PICO = ( 1428 # Skip gn_teensy_build if running on mac-arm64. 1429 # There are no arm specific tools packages available upstream: 1430 # https://www.pjrc.com/teensy/package_teensy_index.json 1431 gn_teensy_build 1432 if not (sys.platform == 'darwin' and platform.machine() == 'arm64') 1433 else (), 1434 gn_pico_build, 1435 gn_pw_system_demo_build, 1436) 1437 1438INTERNAL = (gn_mimxrt595_build, gn_mimxrt595_freertos_build) 1439 1440# The misc program differs from other_checks in that checks in the misc 1441# program block CQ on Linux. 1442MISC = ( 1443 # keep-sorted: start 1444 gn_chre_googletest_nanopb_sapphire_build, 1445 # keep-sorted: end 1446) 1447 1448SANITIZERS = (cpp_checks.all_sanitizers(),) 1449 1450SECURITY = ( 1451 # keep-sorted: start 1452 gn_crypto_mbedtls_build, 1453 gn_crypto_micro_ecc_build, 1454 gn_software_update_build, 1455 # keep-sorted: end 1456) 1457 1458FUZZ = (gn_fuzz_build, oss_fuzz_build) 1459 1460# Avoid running all checks on specific paths. 1461PATH_EXCLUSIONS = FormatOptions.load().exclude 1462 1463_LINTFORMAT = ( 1464 commit_message_format, 1465 copyright_notice, 1466 format_code.presubmit_checks(), 1467 inclusive_language.presubmit_check.with_filter( 1468 exclude=( 1469 r'\byarn.lock$', 1470 r'\bpackage-lock.json$', 1471 ) 1472 ), 1473 cpp_checks.pragma_once, 1474 build.bazel_lint, 1475 owners_lint_checks, 1476 source_in_build.gn(SOURCE_FILES_FILTER).with_file_filter( 1477 SOURCE_FILES_FILTER_GN_EXCLUDE 1478 ), 1479 source_is_in_cmake_build_warn_only, 1480 javascript_checks.eslint if shutil.which('npm') else (), 1481 json_check.presubmit_check, 1482 keep_sorted.presubmit_check, 1483 todo_check_with_exceptions, 1484) 1485 1486LINTFORMAT = ( 1487 _LINTFORMAT, 1488 # This check is excluded from _LINTFORMAT because it's not quick: it issues 1489 # a bazel query that pulls in all of Pigweed's external dependencies 1490 # (https://stackoverflow.com/q/71024130/1224002). These are cached, but 1491 # after a roll it can be quite slow. 1492 source_in_build.bazel(SOURCE_FILES_FILTER), 1493 python_checks.check_python_versions, 1494 python_checks.gn_python_lint, 1495) 1496 1497QUICK = ( 1498 _LINTFORMAT, 1499 gn_quick_build_check, 1500) 1501 1502FULL = ( 1503 _LINTFORMAT, 1504 gn_combined_build_check, 1505 gn_host_tools, 1506 bazel_test, 1507 bazel_build, 1508 python_checks.gn_python_check, 1509 python_checks.gn_python_test_coverage, 1510 python_checks.check_upstream_python_constraints, 1511 build_env_setup, 1512) 1513 1514PROGRAMS = Programs( 1515 # keep-sorted: start 1516 arduino_pico=ARDUINO_PICO, 1517 full=FULL, 1518 fuzz=FUZZ, 1519 internal=INTERNAL, 1520 lintformat=LINTFORMAT, 1521 misc=MISC, 1522 other_checks=OTHER_CHECKS, 1523 quick=QUICK, 1524 sanitizers=SANITIZERS, 1525 security=SECURITY, 1526 # keep-sorted: end 1527) 1528 1529 1530def parse_args() -> argparse.Namespace: 1531 """Creates an argument parser and parses arguments.""" 1532 1533 parser = argparse.ArgumentParser(description=__doc__) 1534 cli.add_arguments(parser, PROGRAMS, 'quick') 1535 parser.add_argument( 1536 '--install', 1537 action='store_true', 1538 help='Install the presubmit as a Git pre-push hook and exit.', 1539 ) 1540 1541 return parser.parse_args() 1542 1543 1544def run(install: bool, exclude: list, **presubmit_args) -> int: 1545 """Entry point for presubmit.""" 1546 1547 if install: 1548 install_git_hook( 1549 'pre-push', 1550 [ 1551 'python', 1552 '-m', 1553 'pw_presubmit.pigweed_presubmit', 1554 '--base', 1555 'origin/main..HEAD', 1556 '--program', 1557 'quick', 1558 ], 1559 ) 1560 return 0 1561 1562 exclude.extend(PATH_EXCLUSIONS) 1563 return cli.run(exclude=exclude, **presubmit_args) 1564 1565 1566def main() -> int: 1567 """Run the presubmit for the Pigweed repository.""" 1568 return run(**vars(parse_args())) 1569 1570 1571if __name__ == '__main__': 1572 try: 1573 # If pw_cli is available, use it to initialize logs. 1574 from pw_cli import log # pylint: disable=ungrouped-imports 1575 1576 log.install(logging.INFO) 1577 except ImportError: 1578 # If pw_cli isn't available, display log messages like a simple print. 1579 logging.basicConfig(format='%(message)s', level=logging.INFO) 1580 1581 sys.exit(main()) 1582