1# Copyright 2021 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/error.gni") 18import("$dir_pw_build/python.gni") 19import("$dir_pw_build/python_action.gni") 20import("$dir_pw_build/python_gn_args.gni") 21import("$dir_pw_build/zip.gni") 22 23# Builds a directory containing a collection of Python wheels. 24# 25# Given one or more pw_python_package targets, this target will build their 26# .wheel sub-targets along with the .wheel sub-targets of all dependencies, 27# direct and indirect, as understood by GN. The resulting .whl files will be 28# collected into a single directory called 'python_wheels'. 29# 30# Args: 31# packages: A list of pw_python_package targets whose wheels should be 32# included; their dependencies will be pulled in as wheels also. 33# directory: output directory for the wheels; defaults to 34# $target_out_dir/$target_name 35# deps: additional dependencies 36# 37template("pw_python_wheels") { 38 _wheel_paths_path = "${target_gen_dir}/${target_name}_wheel_paths.txt" 39 40 _deps = [] 41 if (defined(invoker.deps)) { 42 _deps = invoker.deps 43 } 44 45 if (defined(invoker.directory)) { 46 _directory = invoker.directory 47 } else { 48 _directory = "$target_out_dir/$target_name" 49 } 50 51 _packages = [] 52 foreach(_pkg, invoker.packages) { 53 _pkg_name = get_label_info(_pkg, "label_no_toolchain") 54 _pkg_toolchain = get_label_info(_pkg, "toolchain") 55 _packages += [ "${_pkg_name}.wheel(${_pkg_toolchain})" ] 56 } 57 58 if (defined(invoker.venv)) { 59 _venv_target_label = pw_build_PYTHON_BUILD_VENV 60 _venv_target_label = invoker.venv 61 _venv_target_label = 62 get_label_info(_venv_target_label, "label_no_toolchain") 63 _packages += 64 [ "${_venv_target_label}.vendor_wheels($pw_build_PYTHON_TOOLCHAIN)" ] 65 } 66 67 # Build a list of relative paths containing all the wheels we depend on. 68 generated_file("${target_name}._wheel_paths") { 69 data_keys = [ "pw_python_package_wheels" ] 70 rebase = root_build_dir 71 deps = _packages 72 outputs = [ _wheel_paths_path ] 73 } 74 75 pw_python_action(target_name) { 76 forward_variables_from(invoker, [ "public_deps" ]) 77 deps = _deps + [ ":$target_name._wheel_paths" ] 78 module = "pw_build.collect_wheels" 79 python_deps = [ "$dir_pw_build/py" ] 80 81 args = [ 82 "--prefix", 83 rebase_path(root_build_dir, root_build_dir), 84 "--suffix", 85 rebase_path(_wheel_paths_path, root_build_dir), 86 "--out-dir", 87 rebase_path(_directory, root_build_dir), 88 ] 89 90 stamp = true 91 } 92} 93 94# Builds a .zip containing Python wheels and setup scripts. 95# 96# The resulting .zip archive will contain a directory with Python wheels for 97# all pw_python_package targets listed in 'packages', plus wheels for any 98# pw_python_package targets those packages depend on, directly or indirectly, 99# as understood by GN. 100# 101# In addition to Python wheels, the resulting .zip will also contain simple 102# setup scripts for Linux, MacOS, and Windows that take care of creating a 103# Python venv and installing all the included wheels into it, and a README.md 104# file with setup and usage instructions. 105# 106# Args: 107# packages: A list of pw_python_package targets whose wheels should be 108# included; their dependencies will be pulled in as wheels also. 109# inputs: An optional list of extra files to include in the generated .zip, 110# formatted the same was as the 'inputs' argument to pw_zip targets. 111# dirs: An optional list of directories to include in the generated .zip, 112# formatted the same way as the 'dirs' argument to pw_zip targets. 113template("pw_python_zip_with_setup") { 114 _outer_name = target_name 115 _zip_path = "${target_out_dir}/${target_name}.zip" 116 117 _inputs = [] 118 if (defined(invoker.inputs)) { 119 _inputs = invoker.inputs 120 } 121 _dirs = [] 122 if (defined(invoker.dirs)) { 123 _dirs = invoker.dirs 124 } 125 _public_deps = [] 126 if (defined(invoker.public_deps)) { 127 _public_deps = invoker.public_deps 128 } 129 130 pw_python_wheels("$target_name.wheels") { 131 packages = invoker.packages 132 forward_variables_from(invoker, 133 [ 134 "deps", 135 "venv", 136 ]) 137 } 138 139 pw_zip(target_name) { 140 forward_variables_from(invoker, [ "deps" ]) 141 inputs = _inputs + [ 142 "$dir_pw_build/python_dist/setup.bat > /${target_name}/", 143 "$dir_pw_build/python_dist/setup.sh > /${target_name}/", 144 ] 145 146 dirs = _dirs + [ "$target_out_dir/$target_name.wheels/ > /$target_name/python_wheels/" ] 147 148 output = _zip_path 149 150 # TODO: b/235245034 - Remove the plumbing-through of invoker's public_deps. 151 public_deps = _public_deps + [ ":${_outer_name}.wheels" ] 152 153 if (defined(invoker.venv)) { 154 _venv_target_label = get_label_info(invoker.venv, "label_no_toolchain") 155 _requirements_target_name = 156 get_label_info("${_venv_target_label}($pw_build_PYTHON_TOOLCHAIN)", 157 "name") 158 _requirements_gen_dir = 159 get_label_info("${_venv_target_label}($pw_build_PYTHON_TOOLCHAIN)", 160 "target_gen_dir") 161 162 inputs += [ "$_requirements_gen_dir/$_requirements_target_name/compiled_requirements.txt > /${target_name}/requirements.txt" ] 163 164 public_deps += [ "${_venv_target_label}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)" ] 165 } 166 } 167} 168 169# Generates a directory of Python packages from source files suitable for 170# deployment outside of the project developer environment. 171# 172# The resulting directory contains only files mentioned in each package's 173# setup.cfg file. This is useful for bundling multiple Python packages up 174# into a single package for distribution to other locations like 175# http://pypi.org. 176# 177# Args: 178# packages: A list of pw_python_package targets to be installed into the build 179# directory. Their dependencies will be pulled in as wheels also. 180# 181# include_tests: If true, copy Python package tests to a `tests` subdir. 182# 183# extra_files: A list of extra files that should be included in the output. The 184# format of each item in this list follows this convention: 185# //some/nested/source_file > nested/destination_file 186# 187# generate_setup_cfg: A scope containing either common_config_file or 'name' 188# and 'version' If included this creates a merged setup.cfg for all python 189# Packages using either a common_config_file as a base or name and version 190# strings. This scope can optionally include: 191# 192# include_default_pyproject_file: Include a standard pyproject.toml file 193# that uses setuptools. 194# 195# append_git_sha_to_version: Append the current git SHA to the package 196# version string after a + sign. 197# 198# append_date_to_version: Append the current date to the package version 199# string after a + sign. 200# 201# include_extra_files_in_package_data: Add any extra_files to the setup.cfg 202# file under the [options.package_data] section. 203# 204# auto_create_package_data_init_py_files: Default: true 205# Create __init__.py files as needed in all subdirs of extra_files when 206# including in [options.package_data]. 207# 208template("pw_python_distribution") { 209 _metadata_path_list_suffix = "_pw_python_distribution_metadata_path_list.txt" 210 _output_dir = "${target_out_dir}/${target_name}/" 211 _metadata_json_file_list = 212 "${target_gen_dir}/${target_name}${_metadata_path_list_suffix}" 213 214 # If generating a setup.cfg file a common base file must be provided. 215 if (defined(invoker.generate_setup_cfg)) { 216 generate_setup_cfg = invoker.generate_setup_cfg 217 assert( 218 defined(generate_setup_cfg.common_config_file) || 219 (defined(generate_setup_cfg.name) && 220 defined(generate_setup_cfg.version)), 221 "Either 'common_config_file' or ('name' + 'version') are required in generate_setup_cfg") 222 } 223 224 _extra_file_inputs = [] 225 _extra_file_args = [] 226 227 # Convert extra_file strings to input, outputs and create_python_tree.py args. 228 if (defined(invoker.extra_files)) { 229 _delimiter = ">" 230 _extra_file_outputs = [] 231 foreach(input, invoker.extra_files) { 232 # Remove spaces before and after the delimiter 233 input = string_replace(input, " $_delimiter", _delimiter) 234 input = string_replace(input, "$_delimiter ", _delimiter) 235 236 input_list = [] 237 input_list = string_split(input, _delimiter) 238 239 # Save the input file 240 _extra_file_inputs += [ input_list[0] ] 241 242 # Save the output file 243 _this_output = _output_dir + "/" + input_list[1] 244 _extra_file_outputs += [ _this_output ] 245 246 # Compose an arg for passing to create_python_tree.py with properly 247 # rebased paths. 248 _extra_file_args += 249 [ string_join(" $_delimiter ", 250 [ 251 rebase_path(input_list[0], root_build_dir), 252 rebase_path(_this_output, root_build_dir), 253 ]) ] 254 } 255 } 256 257 _include_tests = defined(invoker.include_tests) && invoker.include_tests 258 259 _public_deps = [] 260 if (defined(invoker.public_deps)) { 261 _public_deps += invoker.public_deps 262 } 263 264 # Set source files for the Python package metadata json file. 265 _sources = [] 266 _setup_sources = [ 267 "$_output_dir/pyproject.toml", 268 "$_output_dir/setup.cfg", 269 ] 270 _test_sources = [] 271 272 # Create the Python package_metadata.json file so this can be used as a 273 # Python dependency. 274 _package_metadata_json_file = 275 "$target_gen_dir/$target_name/package_metadata.json" 276 277 # Get Python package metadata and write to disk as JSON. 278 _package_metadata = { 279 gn_target_name = 280 get_label_info(":${invoker.target_name}", "label_no_toolchain") 281 282 # Get package source files 283 sources = rebase_path(_sources, root_build_dir) 284 285 # Get setup.cfg, pyproject.toml, or setup.py file 286 setup_sources = rebase_path(_setup_sources, root_build_dir) 287 288 # Get test source files 289 tests = rebase_path(_test_sources, root_build_dir) 290 291 # Get package input files (package data) 292 inputs = [] 293 if (defined(invoker.inputs)) { 294 inputs = rebase_path(invoker.inputs, root_build_dir) 295 } 296 inputs += rebase_path(_extra_file_inputs, root_build_dir) 297 } 298 299 # Finally, write out the json 300 write_file(_package_metadata_json_file, _package_metadata, "json") 301 302 group("$target_name._package_metadata") { 303 metadata = { 304 pw_python_package_metadata_json = [ _package_metadata_json_file ] 305 } 306 307 # Forward the package_metadata subtarget for all packages bundled in this 308 # distribution. 309 public_deps = [] 310 foreach(dep, invoker.packages) { 311 public_deps += [ get_label_info(dep, "label_no_toolchain") + 312 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 313 } 314 } 315 316 _package_metadata_targets = [] 317 foreach(pkg, invoker.packages) { 318 _package_metadata_targets += 319 [ get_label_info(pkg, "label_no_toolchain") + 320 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 321 } 322 323 # Build a list of relative paths containing all the python 324 # package_metadata.json files we depend on. 325 generated_file("${target_name}.${_metadata_path_list_suffix}") { 326 data_keys = [ "pw_python_package_metadata_json" ] 327 rebase = root_build_dir 328 deps = _package_metadata_targets 329 outputs = [ _metadata_json_file_list ] 330 } 331 332 # Run the python action on the metadata_path_list.txt file 333 pw_python_action(target_name) { 334 # Save the Python package metadata so this can be installed using 335 # pw_internal_pip_install. 336 metadata = { 337 pw_python_package_metadata_json = [ _package_metadata_json_file ] 338 } 339 340 deps = invoker.packages + 341 [ ":${invoker.target_name}.${_metadata_path_list_suffix}" ] 342 343 script = "$dir_pw_build/py/pw_build/create_python_tree.py" 344 inputs = _extra_file_inputs 345 public_deps = _public_deps 346 _pw_internal_run_in_venv = false 347 348 args = [ 349 "--repo-root", 350 rebase_path("//", root_build_dir), 351 "--tree-destination-dir", 352 rebase_path(_output_dir, root_build_dir), 353 "--input-list-files", 354 rebase_path(_metadata_json_file_list, root_build_dir), 355 ] 356 357 # Add required setup.cfg args if we are generating a merged config. 358 if (defined(generate_setup_cfg)) { 359 if (defined(generate_setup_cfg.common_config_file)) { 360 args += [ 361 "--setupcfg-common-file", 362 rebase_path(generate_setup_cfg.common_config_file, root_build_dir), 363 ] 364 } 365 if (defined(generate_setup_cfg.append_git_sha_to_version)) { 366 args += [ "--setupcfg-version-append-git-sha" ] 367 } 368 if (defined(generate_setup_cfg.append_date_to_version)) { 369 args += [ "--setupcfg-version-append-date" ] 370 } 371 if (defined(generate_setup_cfg.name)) { 372 args += [ 373 "--setupcfg-override-name", 374 generate_setup_cfg.name, 375 ] 376 } 377 if (defined(generate_setup_cfg.version)) { 378 args += [ 379 "--setupcfg-override-version", 380 generate_setup_cfg.version, 381 ] 382 } 383 if (defined(generate_setup_cfg.include_default_pyproject_file) && 384 generate_setup_cfg.include_default_pyproject_file == true) { 385 args += [ "--create-default-pyproject-toml" ] 386 } 387 if (defined(generate_setup_cfg.include_extra_files_in_package_data)) { 388 args += [ "--setupcfg-extra-files-in-package-data" ] 389 } 390 _auto_create_package_data_init_py_files = true 391 if (defined(generate_setup_cfg.auto_create_package_data_init_py_files)) { 392 _auto_create_package_data_init_py_files = 393 generate_setup_cfg.auto_create_package_data_init_py_files 394 } 395 if (_auto_create_package_data_init_py_files) { 396 args += [ "--auto-create-package-data-init-py-files" ] 397 } 398 } 399 400 if (_extra_file_args == []) { 401 # No known output files - stamp instead. 402 stamp = true 403 } else { 404 args += [ "--extra-files" ] 405 args += _extra_file_args 406 407 # Include extra_files as outputs 408 outputs = _extra_file_outputs 409 } 410 411 if (_include_tests) { 412 args += [ "--include-tests" ] 413 } 414 } 415 416 # Template to build a bundled Python package wheel. 417 pw_python_action("$target_name._build_wheel") { 418 _wheel_out_dir = "$target_out_dir/$target_name" 419 _wheel_requirement = "$_wheel_out_dir/requirements.txt" 420 metadata = { 421 pw_python_package_wheels = [ _wheel_out_dir ] 422 } 423 424 script = "$dir_pw_build/py/pw_build/generate_python_wheel.py" 425 426 args = [ 427 "--package-dir", 428 rebase_path(_output_dir, root_build_dir), 429 "--out-dir", 430 rebase_path(_wheel_out_dir, root_build_dir), 431 ] 432 433 # Add hashes to the _wheel_requirement output. 434 if (pw_build_PYTHON_PIP_INSTALL_REQUIRE_HASHES) { 435 args += [ "--generate-hashes" ] 436 } 437 438 public_deps = [] 439 if (defined(invoker.public_deps)) { 440 public_deps += invoker.public_deps 441 } 442 public_deps += [ ":${invoker.target_name}" ] 443 444 outputs = [ _wheel_requirement ] 445 } 446 group("$target_name.wheel") { 447 public_deps = [ ":${invoker.target_name}._build_wheel" ] 448 } 449 450 # Allow using pw_python_distribution targets as a python_dep in 451 # pw_python_group. To do this, create a pw_python_group with the relevant 452 # packages and create wrappers for each subtarget, except those that are 453 # actually implemented by this template. 454 # 455 # This is an ugly workaround that will be removed when the Python build is 456 # refactored (b/235278298). 457 pw_python_group("$target_name._pw_python_group") { 458 python_deps = invoker.packages 459 } 460 461 wrapped_subtargets = pw_python_package_subtargets - [ 462 "wheel", 463 "_build_wheel", 464 ] 465 466 foreach(subtarget, wrapped_subtargets) { 467 group("$target_name.$subtarget") { 468 public_deps = [ ":${invoker.target_name}._pw_python_group.$subtarget" ] 469 } 470 } 471} 472 473# TODO: b/232800695 - Remove this template when all projects no longer use it. 474template("pw_create_python_source_tree") { 475 pw_python_distribution("$target_name") { 476 forward_variables_from(invoker, "*") 477 } 478} 479 480# Runs pip install on a set of pw_python_packages. This will install 481# pw_python_packages into the user's developer environment. 482# 483# Args: 484# packages: A list of pw_python_package targets to be pip installed. 485# These will be installed one at a time. 486# 487# editable: If true, --editable is passed to the pip install command. 488# 489# force_reinstall: If true, --force-reinstall is passed to the pip install 490# command. 491template("pw_python_pip_install") { 492 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 493 # Create a target group for the Python package metadata only. 494 group("$target_name._package_metadata") { 495 # Forward the package_metadata subtarget for all python_deps. 496 public_deps = [] 497 if (defined(invoker.packages)) { 498 foreach(dep, invoker.packages) { 499 public_deps += [ get_label_info(dep, "label_no_toolchain") + 500 "._package_metadata($pw_build_PYTHON_TOOLCHAIN)" ] 501 } 502 } 503 } 504 505 pw_python_action("$target_name") { 506 script = "$dir_pw_build/py/pw_build/pip_install_python_deps.py" 507 508 assert( 509 defined(invoker.packages), 510 "packages = [ 'python_package' ] is required by pw_internal_pip_install") 511 512 public_deps = [] 513 if (defined(invoker.public_deps)) { 514 public_deps += invoker.public_deps 515 } 516 517 python_deps = [] 518 python_metadata_deps = [] 519 if (defined(invoker.packages)) { 520 public_deps += invoker.packages 521 python_deps += invoker.packages 522 python_metadata_deps += invoker.packages 523 } 524 525 python_deps = [] 526 if (defined(invoker.python_deps)) { 527 python_deps += invoker.python_deps 528 } 529 530 _pw_internal_run_in_venv = false 531 _forward_python_metadata_deps = true 532 533 _editable_install = false 534 if (defined(invoker.editable)) { 535 _editable_install = invoker.editable 536 } 537 538 _pkg_gn_labels = [] 539 foreach(pkg, invoker.packages) { 540 _pkg_gn_labels += [ get_label_info(pkg, "label_no_toolchain") ] 541 } 542 543 _pip_install_log_file = "$target_gen_dir/$target_name/pip_install_log.txt" 544 545 args = [ 546 "--gn-packages", 547 string_join(",", _pkg_gn_labels), 548 ] 549 550 if (_editable_install) { 551 args += [ "--editable-pip-install" ] 552 } 553 554 args += [ 555 "--log", 556 rebase_path(_pip_install_log_file, root_build_dir), 557 ] 558 args += pw_build_PYTHON_PIP_DEFAULT_OPTIONS 559 args += [ 560 "install", 561 "--no-build-isolation", 562 ] 563 564 if (!_editable_install) { 565 if (pw_build_PYTHON_PIP_INSTALL_REQUIRE_HASHES) { 566 args += [ "--require-hashes" ] 567 568 # The --require-hashes option can only install wheels via 569 # requirement.txt files that contain hashes. Depend on this package's 570 # _build_wheel target. 571 foreach(pkg, _pkg_gn_labels) { 572 public_deps += [ "${pkg}._build_wheel" ] 573 } 574 } 575 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 576 args += [ "--no-index" ] 577 } 578 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 579 args += [ "--no-cache-dir" ] 580 } 581 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 582 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 583 args += [ 584 "--find-links", 585 rebase_path(link_dir, root_build_dir), 586 ] 587 } 588 } 589 } 590 591 _force_reinstall = false 592 if (defined(invoker.force_reinstall)) { 593 _force_reinstall = true 594 } 595 if (_force_reinstall) { 596 args += [ "--force-reinstall" ] 597 } 598 599 inputs = pw_build_PIP_CONSTRAINTS 600 601 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) { 602 args += [ 603 "--constraint", 604 rebase_path(_constraints_file, root_build_dir), 605 ] 606 } 607 608 stamp = true 609 610 # Parallel pip installations don't work, so serialize pip invocations. 611 pool = "$dir_pw_build/pool:pip($default_toolchain)" 612 } 613 } else { 614 group("$target_name") { 615 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ] 616 } 617 not_needed("*") 618 not_needed(invoker, "*") 619 } 620 621 group("$target_name.install") { 622 public_deps = [ ":${invoker.target_name}" ] 623 } 624 625 # Allow using pw_internal_pip_install targets as a python_dep in 626 # pw_python_group. To do this, create a pw_python_group with the relevant 627 # packages and create wrappers for each subtarget, except those that are 628 # actually implemented by this template. 629 # 630 # This is an ugly workaround that will be removed when the Python build is 631 # refactored (b/235278298). 632 pw_python_group("$target_name._pw_python_group") { 633 python_deps = invoker.packages 634 } 635 636 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) { 637 group("$target_name.$subtarget") { 638 _test_and_lint_subtargets = [ 639 "tests", 640 "lint", 641 "lint.mypy", 642 "lint.pylint", 643 ] 644 if (pw_build_TEST_TRANSITIVE_PYTHON_DEPS || 645 filter_exclude([ subtarget ], _test_and_lint_subtargets) != []) { 646 public_deps = [ ":${invoker.target_name}._pw_python_group.$subtarget" ] 647 } 648 not_needed([ "_test_and_lint_subtargets" ]) 649 } 650 } 651} 652 653# TODO: b/232800695 - Remove this template when all projects no longer use it. 654template("pw_internal_pip_install") { 655 pw_python_pip_install("$target_name") { 656 forward_variables_from(invoker, "*") 657 } 658} 659