1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/input_group.gni") 18import("$dir_pw_build/mirror_tree.gni") 19import("$dir_pw_build/python_action.gni") 20import("$dir_pw_build/python_gn_args.gni") 21import("$dir_pw_protobuf_compiler/toolchain.gni") 22 23declare_args() { 24 # Constraints file selection (arguments to pip install --constraint). 25 # See pip help install. 26 pw_build_PIP_CONSTRAINTS = 27 [ "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list" ] 28 29 # Default pip requirements file for all Pigweed based projects. 30 pw_build_PIP_REQUIREMENTS = [] 31 32 # If true, GN will run each Python test using the coverage command. A separate 33 # coverage data file for each test will be saved. To generate reports from 34 # this information run: pw presubmit --step gn_python_test_coverage 35 pw_build_PYTHON_TEST_COVERAGE = false 36 37 # Output format for pylint. Options include "text" and "colorized". 38 pw_build_PYLINT_OUTPUT_FORMAT = "colorized" 39 40 # Whether or not to lint/test transitive deps of pw_python_package targets. 41 # 42 # For example: if lib_a depends on lib_b, lib_a.tests will run after first 43 # running lib_b.tests if pw_build_TEST_TRANSITIVE_PYTHON_DEPS is true. 44 # 45 # If pw_build_TEST_TRANSITIVE_PYTHON_DEPS is false, tests for a 46 # pw_python_package will run if you directly build the target (e.g. 47 # lib_b.tests) OR if the pw_python_package is placed in a pw_python_group AND 48 # you build the group.tests target. 49 # 50 # This applies to mypy, pylint, and tests. 51 # 52 # While this defaults to true for compatibility reasons, it's strongly 53 # recommended to turn this off so you're not linting and testing all of your 54 # external dependencies. 55 pw_build_TEST_TRANSITIVE_PYTHON_DEPS = true 56} 57 58# Python packages provide the following targets as $target_name.$subtarget. 59pw_python_package_subtargets = [ 60 "tests", 61 "lint", 62 "lint.mypy", 63 "lint.pylint", 64 "install", 65 "wheel", 66 67 # Internal targets that directly depend on one another. 68 "_build_wheel", 69] 70 71# Create aliases for subsargets when the target name matches the directory name. 72# This allows //foo:foo.tests to be accessed as //foo:tests, for example. 73template("_pw_create_aliases_if_name_matches_directory") { 74 not_needed([ "invoker" ]) 75 76 if (get_label_info(":$target_name", "name") == 77 get_path_info(get_label_info(":$target_name", "dir"), "name")) { 78 foreach(subtarget, pw_python_package_subtargets) { 79 group(subtarget) { 80 public_deps = [ ":${invoker.target_name}.$subtarget" ] 81 } 82 } 83 } 84} 85 86# Internal template that runs Mypy. 87template("_pw_python_static_analysis_mypy") { 88 pw_python_action(target_name) { 89 module = "mypy" 90 91 # DOCSTAG: [default-mypy-args] 92 args = [ 93 "--pretty", 94 "--show-error-codes", 95 96 # Use a mypy cache dir for this target only to avoid cache conflicts in 97 # parallel mypy invocations. 98 "--cache-dir", 99 rebase_path(target_out_dir, root_build_dir) + "/.mypy_cache", 100 ] 101 102 # Use this environment variable to force mypy to colorize output. 103 # See https://github.com/python/mypy/issues/7771 104 environment = [ "MYPY_FORCE_COLOR=1" ] 105 106 # DOCSTAG: [default-mypy-args] 107 108 if (defined(invoker.mypy_ini)) { 109 args += 110 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ] 111 inputs = [ invoker.mypy_ini ] 112 } 113 114 args += rebase_path(invoker.sources, root_build_dir) 115 116 stamp = true 117 118 deps = invoker.deps 119 120 if (defined(invoker.python_deps)) { 121 python_deps = invoker.python_deps 122 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS) { 123 foreach(dep, invoker.python_deps) { 124 deps += [ string_replace(dep, "(", ".lint.mypy(") ] 125 } 126 } 127 } 128 if (defined(invoker.python_metadata_deps)) { 129 python_metadata_deps = invoker.python_metadata_deps 130 } 131 } 132} 133 134# Internal template that runs Pylint. 135template("_pw_python_static_analysis_pylint") { 136 # Create a target to run pylint on each of the Python files in this 137 # package and its dependencies. 138 pw_python_action_foreach(target_name) { 139 module = "pylint" 140 args = [ 141 rebase_path(".", root_build_dir) + "/{{source_target_relative}}", 142 "--jobs=1", 143 "--output-format=$pw_build_PYLINT_OUTPUT_FORMAT", 144 ] 145 146 if (defined(invoker.pylintrc)) { 147 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ] 148 inputs = [ invoker.pylintrc ] 149 } 150 151 if (host_os == "win") { 152 # Allow CRLF on Windows, in case Git is set to switch line endings. 153 args += [ "--disable=unexpected-line-ending-format" ] 154 } 155 156 sources = invoker.sources 157 158 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed" 159 160 public_deps = invoker.deps 161 162 if (defined(invoker.python_deps)) { 163 python_deps = invoker.python_deps 164 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS) { 165 foreach(dep, invoker.python_deps) { 166 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ] 167 } 168 } 169 } 170 if (defined(invoker.python_metadata_deps)) { 171 python_metadata_deps = invoker.python_metadata_deps 172 } 173 } 174} 175 176# Defines a Python package. GN Python packages contain several GN targets: 177# 178# - $name - Provides the Python files in the build, but does not take any 179# actions. All subtargets depend on this target. 180# - $name.lint - Runs static analyis tools on the Python code. This is a group 181# of two subtargets: 182# - $name.lint.mypy - Runs mypy (if enabled). 183# - $name.lint.pylint - Runs pylint (if enabled). 184# - $name.tests - Runs all tests for this package. 185# - $name.install - Installs the package in a venv. 186# - $name.wheel - Builds a Python wheel for the package. 187# 188# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN, 189# regardless of the current toolchain. This prevents Python-specific work, like 190# running Pylint, from occurring multiple times in a build. 191# 192# Args: 193# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg), 194# which must all be in the same directory. 195# generate_setup: As an alternative to 'setup', generate setup files with the 196# keywords in this scope. 'name' is required. 197# sources: Python sources files in the package. 198# tests: Test files for this Python package. 199# python_deps: Dependencies on other pw_python_packages in the GN build. 200# python_test_deps: Test-only pw_python_package dependencies. 201# other_deps: Dependencies on GN targets that are not pw_python_packages. 202# inputs: Other files to track, such as package_data. 203# proto_library: A pw_proto_library target to embed in this Python package. 204# generate_setup is required in place of setup if proto_library is used. 205# static_analysis: List of static analysis tools to run; "*" (default) runs 206# all tools. The supported tools are "mypy" and "pylint". 207# pylintrc: Path to a pylintrc configuration file to use. If not 208# provided, Pylint's default rcfile search is used. As this may 209# use the the local user's configuration file, it is highly 210# recommended to pass this option to specify the rcfile explicitly. 211# mypy_ini: Optional path to a mypy configuration file to use. If not 212# provided, mypy's default configuration file search is used. mypy is 213# executed from the package's setup directory, so mypy.ini files in that 214# directory will take precedence over others. 215# 216template("pw_python_package") { 217 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use 218 # fully qualified labels so that the toolchain is not lost. 219 _other_deps = [] 220 if (defined(invoker.other_deps)) { 221 foreach(dep, invoker.other_deps) { 222 _other_deps += [ get_label_info(dep, "label_with_toolchain") ] 223 } 224 } 225 226 _python_deps = [] 227 if (defined(invoker.python_deps)) { 228 foreach(dep, invoker.python_deps) { 229 _python_deps += [ get_label_info(dep, "label_with_toolchain") ] 230 } 231 } 232 233 # pw_python_script uses pw_python_package, but with a limited set of features. 234 # _pw_standalone signals that this target is actually a pw_python_script. 235 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone) 236 237 _generate_package = false 238 239 _pydeplabel = get_label_info(":$target_name", "label_with_toolchain") 240 241 # If a package does not run static analysis or if it does but doesn't have 242 # any tests then this variable is not used. 243 not_needed([ "_pydeplabel" ]) 244 245 # Check the generate_setup and import_protos args to determine if this package 246 # is generated. 247 if (_is_package) { 248 assert(defined(invoker.generate_setup) != defined(invoker.setup), 249 "Either 'setup' or 'generate_setup' (but not both) must provided") 250 251 if (defined(invoker.proto_library)) { 252 assert(invoker.proto_library != "", "'proto_library' cannot be empty") 253 assert(defined(invoker.generate_setup), 254 "Python packages that import protos with 'proto_library' must " + 255 "use 'generate_setup' instead of 'setup'") 256 257 _import_protos = [ invoker.proto_library ] 258 259 # Depend on the dependencies of the proto library. 260 _proto = get_label_info(invoker.proto_library, "label_no_toolchain") 261 _toolchain = get_label_info(invoker.proto_library, "toolchain") 262 _python_deps += [ "$_proto.python._deps($_toolchain)" ] 263 } else if (defined(invoker.generate_setup)) { 264 _import_protos = [] 265 } 266 267 if (defined(invoker.generate_setup)) { 268 _generate_package = true 269 _setup_dir = "$target_gen_dir/$target_name.generated_python_package" 270 271 if (defined(invoker.strip_prefix)) { 272 _source_root = invoker.strip_prefix 273 } else { 274 _source_root = "." 275 } 276 } else { 277 # Non-generated packages with sources provided need an __init__.py. 278 assert(!defined(invoker.sources) || invoker.sources == [] || 279 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [], 280 "Python packages must have at least one __init__.py file") 281 282 # Get the directories of the setup files. All must be in the same dir. 283 _setup_dirs = get_path_info(invoker.setup, "dir") 284 _setup_dir = _setup_dirs[0] 285 286 foreach(dir, _setup_dirs) { 287 assert(dir == _setup_dir, 288 "All files in 'setup' must be in the same directory") 289 } 290 291 assert(!defined(invoker.strip_prefix), 292 "'strip_prefix' may only be given if 'generate_setup' is provided") 293 } 294 } 295 296 # Process arguments defaults and set defaults. 297 298 _supported_static_analysis_tools = [ 299 "mypy", 300 "pylint", 301 ] 302 not_needed([ "_supported_static_analysis_tools" ]) 303 304 # Argument: static_analysis (list of tool names or "*"); default = "*" (all) 305 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") { 306 _static_analysis = _supported_static_analysis_tools 307 } else { 308 _static_analysis = invoker.static_analysis 309 } 310 311 foreach(_tool, _static_analysis) { 312 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] != 313 _supported_static_analysis_tools, 314 "'$_tool' is not a supported static analysis tool") 315 } 316 317 # Argument: sources (list) 318 _sources = [] 319 if (defined(invoker.sources)) { 320 if (_generate_package) { 321 foreach(source, rebase_path(invoker.sources, _source_root)) { 322 _sources += [ "$_setup_dir/$source" ] 323 } 324 } else { 325 _sources += invoker.sources 326 } 327 } 328 329 # Argument: tests (list) 330 _test_sources = [] 331 if (defined(invoker.tests)) { 332 if (_generate_package) { 333 foreach(source, rebase_path(invoker.tests, _source_root)) { 334 _test_sources += [ "$_setup_dir/$source" ] 335 } 336 } else { 337 _test_sources += invoker.tests 338 } 339 } 340 341 # Argument: setup (list) 342 _setup_sources = [] 343 if (defined(invoker.setup)) { 344 _setup_sources = invoker.setup 345 } else if (_generate_package) { 346 _setup_sources = [ 347 "$_setup_dir/pyproject.toml", 348 "$_setup_dir/setup.cfg", 349 ] 350 } 351 352 # Argument: python_test_deps (list) 353 _python_test_deps = _python_deps # include all deps in test deps 354 if (defined(invoker.python_test_deps)) { 355 foreach(dep, invoker.python_test_deps) { 356 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ] 357 } 358 } 359 360 if (_test_sources == []) { 361 assert(!defined(invoker.python_test_deps), 362 "python_test_deps was provided, but there are no tests in " + 363 get_label_info(":$target_name", "label_no_toolchain")) 364 not_needed([ "_python_test_deps" ]) 365 } 366 367 _all_py_files = 368 _sources + _test_sources + filter_include(_setup_sources, [ "*.py" ]) 369 370 # The pw_python_package subtargets are only instantiated in 371 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the 372 # targets in this toolchain. 373 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 374 # Create the package_metadata.json file. This is used by the 375 # pw_python_distribution template. 376 _package_metadata_json_file = 377 "$target_gen_dir/$target_name/package_metadata.json" 378 379 # Get Python package metadata and write to disk as JSON. 380 _package_metadata = { 381 gn_target_name = 382 get_label_info(":${invoker.target_name}", "label_no_toolchain") 383 384 # Get package source files 385 sources = rebase_path(_sources, root_build_dir) 386 387 # Get setup.cfg, pyproject.toml, or setup.py file 388 setup_sources = rebase_path(_setup_sources, root_build_dir) 389 390 # Get test source files 391 tests = rebase_path(_test_sources, root_build_dir) 392 393 # Get package input files (package data) 394 inputs = [] 395 if (defined(invoker.inputs)) { 396 inputs = rebase_path(invoker.inputs, root_build_dir) 397 } 398 399 # Get generate_setup 400 if (defined(invoker.generate_setup)) { 401 generate_setup = invoker.generate_setup 402 } 403 } 404 405 # Finally, write out the json 406 write_file(_package_metadata_json_file, _package_metadata, "json") 407 408 # Create a target group for the Python package metadata only. This is a 409 # python_action so the setup sources can be included as inputs. 410 pw_python_action("$target_name._package_metadata") { 411 metadata = { 412 pw_python_package_metadata_json = [ _package_metadata_json_file ] 413 } 414 415 script = "$dir_pw_build/py/pw_build/nop.py" 416 417 if (_generate_package) { 418 inputs = [ "$_setup_dir/setup.json" ] 419 } else { 420 inputs = _setup_sources 421 } 422 423 _pw_internal_run_in_venv = false 424 stamp = true 425 426 # Forward the package_metadata subtarget for all python_deps. 427 public_deps = [] 428 foreach(dep, _python_deps) { 429 public_deps += [ get_label_info(dep, "label_no_toolchain") + 430 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 431 } 432 } 433 434 # Declare the main Python package group. This represents the Python files, 435 # but does not take any actions. GN targets can depend on the package name 436 # to run when any files in the package change. 437 if (_generate_package) { 438 # If this package is generated, mirror the sources to the final directory. 439 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") { 440 directory = _setup_dir 441 442 sources = [] 443 if (defined(invoker.sources)) { 444 sources += invoker.sources 445 } 446 if (defined(invoker.tests)) { 447 sources += invoker.tests 448 } 449 if (defined(invoker.inputs)) { 450 sources += invoker.inputs 451 } 452 453 source_root = _source_root 454 public_deps = _python_deps + _other_deps 455 } 456 457 # Get generated_setup scope and write it to disk as JSON. 458 459 # Expected setup.cfg structure: 460 # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html 461 _gen_setup = invoker.generate_setup 462 assert(defined(_gen_setup.metadata), 463 "'metadata = {}' is required in generate_package") 464 465 # Get metadata which should contain at least name. 466 _gen_metadata = { 467 } 468 _gen_metadata = _gen_setup.metadata 469 assert( 470 defined(_gen_metadata.name), 471 "metadata = { name = 'package_name' } is required in generate_package") 472 473 # Get options which should not have packages or package_data. 474 if (defined(_gen_setup.options)) { 475 _gen_options = { 476 } 477 _gen_options = _gen_setup.options 478 assert(!defined(_gen_options.packages) && 479 !defined(_gen_options.package_data), 480 "'packages' and 'package_data' may not be provided " + 481 "in 'generate_package' options.") 482 } 483 484 write_file("$_setup_dir/setup.json", _gen_setup, "json") 485 486 # Generate the setup.py, py.typed, and __init__.py files as needed. 487 action(target_name) { 488 metadata = { 489 pw_python_package_metadata_json = [ _package_metadata_json_file ] 490 } 491 492 script = "$dir_pw_build/py/pw_build/generate_python_package.py" 493 args = [ 494 "--label", 495 get_label_info(":$target_name", "label_no_toolchain"), 496 "--generated-root", 497 rebase_path(_setup_dir, root_build_dir), 498 "--setup-json", 499 rebase_path("$_setup_dir/setup.json", root_build_dir), 500 ] + rebase_path(_sources, root_build_dir) 501 502 # Pass in the .json information files for the imported proto libraries. 503 foreach(proto, _import_protos) { 504 _label = get_label_info(proto, "label_no_toolchain") + 505 ".python($pw_protobuf_compiler_TOOLCHAIN)" 506 _file = get_label_info(_label, "target_gen_dir") + "/" + 507 get_label_info(_label, "name") + ".json" 508 args += [ 509 "--proto-library", 510 rebase_path(_file, root_build_dir), 511 ] 512 } 513 514 if (defined(invoker._pw_module_as_package) && 515 invoker._pw_module_as_package) { 516 args += [ "--module-as-package" ] 517 } 518 519 inputs = [ "$_setup_dir/setup.json" ] 520 521 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ] 522 523 outputs = _setup_sources 524 } 525 } else { 526 # If the package is not generated, use an input group for the sources. 527 pw_input_group(target_name) { 528 metadata = { 529 pw_python_package_metadata_json = [ _package_metadata_json_file ] 530 } 531 inputs = _all_py_files 532 if (defined(invoker.inputs)) { 533 inputs += invoker.inputs 534 } 535 536 public_deps = _python_deps + _other_deps 537 } 538 } 539 540 if (_is_package) { 541 # Builds a Python wheel for this package. Records the output directory 542 # in the pw_python_package_wheels metadata key. 543 544 pw_python_action("$target_name._build_wheel") { 545 _wheel_out_dir = "$target_out_dir/$target_name" 546 _wheel_requirement = "$_wheel_out_dir/requirements.txt" 547 metadata = { 548 pw_python_package_wheels = [ _wheel_out_dir ] 549 } 550 551 script = "$dir_pw_build/py/pw_build/generate_python_wheel.py" 552 553 args = [ 554 "--package-dir", 555 rebase_path(_setup_dir, root_build_dir), 556 "--out-dir", 557 rebase_path(_wheel_out_dir, root_build_dir), 558 ] 559 560 # Add hashes to the _wheel_requirement output. 561 if (pw_build_PYTHON_PIP_INSTALL_REQUIRE_HASHES) { 562 args += [ "--generate-hashes" ] 563 } 564 565 deps = [ ":${invoker.target_name}" ] 566 foreach(dep, _python_deps) { 567 deps += [ string_replace(dep, "(", ".wheel(") ] 568 } 569 570 outputs = [ _wheel_requirement ] 571 } 572 } else { 573 # Stub for non-package targets. 574 group("$target_name._build_wheel") { 575 } 576 } 577 578 # Create the .install and .wheel targets. To limit unnecessary pip 579 # executions, non-generated packages are only reinstalled when their 580 # setup.py changes. However, targets that depend on the .install subtarget 581 # re-run whenever any source files change. 582 # 583 # These targets just represent the source files if this isn't a package. 584 group("$target_name.install") { 585 public_deps = [ ":${invoker.target_name}" ] 586 587 foreach(dep, _python_deps) { 588 public_deps += [ string_replace(dep, "(", ".install(") ] 589 } 590 } 591 592 group("$target_name.wheel") { 593 public_deps = [ ":${invoker.target_name}.install" ] 594 595 if (_is_package) { 596 public_deps += [ ":${invoker.target_name}._build_wheel" ] 597 } 598 599 foreach(dep, _python_deps) { 600 public_deps += [ string_replace(dep, "(", ".wheel(") ] 601 } 602 } 603 604 # Define the static analysis targets for this package. 605 group("$target_name.lint") { 606 deps = [] 607 foreach(_tool, _supported_static_analysis_tools) { 608 deps += [ ":${invoker.target_name}.lint.$_tool" ] 609 } 610 } 611 612 if (_static_analysis != [] || _test_sources != []) { 613 # All packages to install for either general use or test running. 614 _test_install_deps = [ ":$target_name.install" ] 615 616 foreach(dep, _python_test_deps) { 617 _test_install_deps += [ string_replace(dep, "(", ".install(") ] 618 _test_install_deps += [ dep ] 619 } 620 } 621 622 # For packages that are not generated, create targets to run mypy and pylint. 623 foreach(_tool, _static_analysis) { 624 # Run lint tools from the setup or target directory so that the tools detect 625 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files 626 # may be explicitly specified with the pylintrc or mypy_ini arguments. 627 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") { 628 sources = _all_py_files 629 deps = _test_install_deps 630 python_deps = _python_deps + _python_test_deps 631 632 if (_is_package) { 633 python_metadata_deps = [ _pydeplabel ] 634 } 635 636 _optional_variables = [ 637 "mypy_ini", 638 "pylintrc", 639 ] 640 forward_variables_from(invoker, _optional_variables) 641 not_needed(_optional_variables) 642 } 643 } 644 645 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) { 646 pw_input_group("$target_name.lint.$_unused_tool") { 647 inputs = [] 648 if (defined(invoker.pylintrc)) { 649 inputs += [ invoker.pylintrc ] 650 } 651 if (defined(invoker.mypy_ini)) { 652 inputs += [ invoker.mypy_ini ] 653 } 654 } 655 656 # Generated packages with linting disabled never need the whole file list. 657 not_needed([ "_all_py_files" ]) 658 } 659 } else { 660 # Create groups with the public target names ($target_name, $target_name.lint, 661 # $target_name.install, etc.). These are actually wrappers around internal 662 # Python actions instantiated with the default toolchain. This ensures there 663 # is only a single copy of each Python action in the build. 664 # 665 # The $target_name.tests group is created separately below. 666 group("$target_name") { 667 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ] 668 } 669 670 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) { 671 group("$target_name.$subtarget") { 672 deps = 673 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ] 674 } 675 } 676 677 # Everything Python-related is only instantiated in the default toolchain. 678 # Silence not-needed warnings except for in the default toolchain. 679 not_needed("*") 680 not_needed(invoker, "*") 681 } 682 683 # Create a target for each test file. 684 _test_targets = [] 685 686 foreach(test, _test_sources) { 687 if (_is_package) { 688 _name = rebase_path(test, _setup_dir) 689 } else { 690 _name = test 691 } 692 693 _test_target = "$target_name.tests." + string_replace(_name, "/", "_") 694 695 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 696 pw_python_action(_test_target) { 697 if (pw_build_PYTHON_TEST_COVERAGE) { 698 module = "coverage" 699 working_directory = 700 rebase_path(get_path_info(test, "dir"), root_build_dir) 701 args = [ 702 "run", 703 "--branch", 704 705 # Include all source files in the working_directory when calculating coverage. 706 "--source=.", 707 708 # Test file to run. 709 get_path_info(test, "file"), 710 ] 711 712 # Set the coverage file to a location in out/python/gen/ 713 _coverage_data_file = "$target_gen_dir/$target_name.coverage" 714 outputs = [ _coverage_data_file ] 715 716 # The coverage tool only allows setting the output with an environment variable. 717 environment = 718 [ "COVERAGE_FILE=" + 719 rebase_path(_coverage_data_file, get_path_info(test, "dir")) ] 720 } else { 721 script = test 722 } 723 724 stamp = true 725 726 # Make sure the python test deps are added to the PYTHONPATH. 727 python_metadata_deps = _python_test_deps 728 729 # If this is a test for a package, add it to PYTHONPATH as well. This is 730 # required if the test source file isn't in the same directory as the 731 # folder containing the package sources to allow local Python imports. 732 if (_is_package) { 733 python_metadata_deps += [ _pydeplabel ] 734 } 735 736 deps = _test_install_deps 737 738 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS) { 739 foreach(dep, _python_test_deps) { 740 deps += [ string_replace(dep, "(", ".tests(") ] 741 } 742 } 743 } 744 } else { 745 # Create a public version of each test target, so tests can be executed as 746 # //path/to:package.tests.foo.py. 747 group(_test_target) { 748 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ] 749 } 750 } 751 752 _test_targets += [ ":$_test_target" ] 753 } 754 755 group("$target_name.tests") { 756 deps = _test_targets 757 } 758 759 _pw_create_aliases_if_name_matches_directory(target_name) { 760 } 761} 762 763# Declares a group of Python packages or other Python groups. pw_python_groups 764# expose the same set of subtargets as pw_python_package (e.g. 765# "$group_name.lint" and "$group_name.tests"), but these apply to all packages 766# in deps and their dependencies. 767template("pw_python_group") { 768 if (defined(invoker.python_deps)) { 769 _python_deps = invoker.python_deps 770 } else { 771 _python_deps = [] 772 not_needed([ "invoker" ]) # Allow empty groups. 773 } 774 775 group(target_name) { 776 deps = _python_deps 777 778 if (defined(invoker.other_deps)) { 779 deps += invoker.other_deps 780 } 781 } 782 783 # Create a target group for the Python package metadata only. 784 group("$target_name._package_metadata") { 785 # Forward the package_metadata subtarget for all python_deps. 786 public_deps = [] 787 foreach(dep, _python_deps) { 788 public_deps += [ get_label_info(dep, "label_no_toolchain") + 789 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 790 } 791 } 792 793 foreach(subtarget, pw_python_package_subtargets) { 794 group("$target_name.$subtarget") { 795 public_deps = [] 796 foreach(dep, _python_deps) { 797 # Split out the toolchain to support deps with a toolchain specified. 798 _target = get_label_info(dep, "label_no_toolchain") 799 _toolchain = get_label_info(dep, "toolchain") 800 public_deps += [ "$_target.$subtarget($_toolchain)" ] 801 } 802 } 803 } 804 805 _pw_create_aliases_if_name_matches_directory(target_name) { 806 } 807} 808 809# Declares Python scripts or tests that are not part of a Python package. 810# Similar to pw_python_package, but only supports a subset of its features. 811# 812# pw_python_script accepts the same arguments as pw_python_package, except 813# `setup` cannot be provided. 814# 815# pw_python_script provides the same subtargets as pw_python_package, but 816# $target_name.install and $target_name.wheel only affect the python_deps of 817# this GN target, not the target itself. 818# 819# pw_python_script allows creating a pw_python_action associated with the 820# script. This is provided by passing an 'action' scope to pw_python_script. 821# This functions like a normal action, with a few additions: the action uses the 822# pw_python_script's python_deps and defaults to using the source file as its 823# 'script' argument, if there is only a single source file. 824template("pw_python_script") { 825 _package_variables = [ 826 "sources", 827 "tests", 828 "python_deps", 829 "python_test_deps", 830 "python_metadata_deps", 831 "other_deps", 832 "inputs", 833 "pylintrc", 834 "mypy_ini", 835 "static_analysis", 836 ] 837 838 pw_python_package(target_name) { 839 _pw_standalone = true 840 forward_variables_from(invoker, _package_variables) 841 } 842 843 _pw_create_aliases_if_name_matches_directory(target_name) { 844 } 845 846 if (defined(invoker.action)) { 847 pw_python_action("$target_name.action") { 848 forward_variables_from(invoker.action, "*", [ "python_deps" ]) 849 forward_variables_from(invoker, [ "testonly" ]) 850 python_deps = [ ":${invoker.target_name}" ] 851 852 if (!defined(script) && !defined(module) && defined(invoker.sources)) { 853 _sources = invoker.sources 854 assert(_sources != [] && _sources == [ _sources[0] ], 855 "'script' must be specified unless there is only one source " + 856 "in 'sources'") 857 script = _sources[0] 858 } 859 } 860 } 861} 862 863# Represents a list of Python requirements, as in a requirements.txt. 864# 865# Args: 866# files: One or more requirements.txt files. 867# requirements: A list of requirements.txt-style requirements. 868template("pw_python_requirements") { 869 assert(defined(invoker.files) || defined(invoker.requirements), 870 "pw_python_requirements requires a list of requirements.txt files " + 871 "in the 'files' arg or requirements in 'requirements'") 872 873 _requirements_files = [] 874 875 if (defined(invoker.files)) { 876 _requirements_files += invoker.files 877 } 878 879 if (defined(invoker.requirements)) { 880 _requirements_file = "$target_gen_dir/$target_name.requirements.txt" 881 write_file(_requirements_file, invoker.requirements) 882 _requirements_files += [ _requirements_file ] 883 } 884 885 # The default target represents the requirements themselves. 886 pw_input_group(target_name) { 887 inputs = _requirements_files 888 } 889 890 # Use the same subtargets as pw_python_package so these targets can be listed 891 # as python_deps of pw_python_packages. 892 group("$target_name.install") { 893 # TODO: b/232800695 - Remove reliance on this subtarget existing. 894 } 895 896 # Create stubs for the unused subtargets so that pw_python_requirements can be 897 # used as python_deps. 898 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) { 899 group("$target_name.$subtarget") { 900 } 901 } 902 903 # Create a target group for the Python package metadata only. 904 group("$target_name._package_metadata") { 905 # Forward the package_metadata subtarget for all python_deps. 906 public_deps = [] 907 if (defined(invoker.python_deps)) { 908 foreach(dep, invoker.python_deps) { 909 public_deps += [ get_label_info(dep, "label_no_toolchain") + 910 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 911 } 912 } 913 } 914 915 _pw_create_aliases_if_name_matches_directory(target_name) { 916 } 917} 918