1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/python.gni") 18import("$dir_pw_build/python_action.gni") 19 20# Defines and creates a Python virtualenv. This template is used by Pigweed in 21# https://cs.pigweed.dev/pigweed/+/main:pw_env_setup/BUILD.gn to 22# create a virtualenv for use within the GN build that all Python actions will 23# run in. 24# 25# Example: 26# 27# pw_python_venv("test_venv") { 28# path = "test-venv" 29# constraints = [ "//tools/constraints.list" ] 30# requirements = [ "//tools/requirements.txt" ] 31# source_packages = [ 32# "$dir_pw_cli/py", 33# "$dir_pw_console/py", 34# "//tools:another_pw_python_package", 35# ] 36# } 37# 38# Args: 39# path: The directory where the virtualenv will be created. This is relative 40# to the GN root and must begin with "$root_build_dir/" if it lives in the 41# output directory or "//" if it lives in elsewhere. 42# 43# constraints: A list of constraint files used when performing pip install 44# into this virtualenv. By default this is set to pw_build_PIP_CONSTRAINTS 45# 46# requirements: A list of requirements files to install into this virtualenv 47# on creation. By default this is set to pw_build_PIP_REQUIREMENTS 48# 49# pip_generate_hashes: (Default: false) Use --generate-hashes When 50# running pip-compile to compute the final requirements.txt 51# 52# source_packages: A list of in-tree pw_python_package targets that will be 53# checked for external third_party pip dependencies to install into this 54# virtualenv. Note this list of targets isn't actually installed into the 55# virtualenv. Only packages defined inside the [options] install_requires 56# section of each pw_python_package's setup.cfg will be pip installed. See 57# this page for a setup.cfg example: 58# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html 59# 60# output_logs: (Default: true) Commands will output logs. 61# 62template("pw_python_venv") { 63 assert(defined(invoker.path), "pw_python_venv requires a 'path'") 64 65 _path = invoker.path 66 67 _generated_requirements_file = 68 "$target_gen_dir/$target_name/generated_requirements.txt" 69 70 _compiled_requirements_file = 71 "$target_gen_dir/$target_name/compiled_requirements.txt" 72 73 _source_packages = [] 74 if (defined(invoker.source_packages)) { 75 _source_packages += invoker.source_packages 76 } else { 77 not_needed([ 78 "_source_packages", 79 "_generated_requirements_file", 80 ]) 81 } 82 _output_logs = true 83 if (defined(invoker.output_logs)) { 84 _output_logs = invoker.output_logs 85 } 86 if (!defined(invoker.output_logs) || 87 current_toolchain != pw_build_PYTHON_TOOLCHAIN) { 88 not_needed([ "_output_logs" ]) 89 } 90 91 _source_package_labels = [] 92 foreach(pkg, _source_packages) { 93 _source_package_labels += [ get_label_info(pkg, "label_no_toolchain") ] 94 } 95 96 if (defined(invoker.requirements)) { 97 _requirements = invoker.requirements 98 } else { 99 _requirements = pw_build_PIP_REQUIREMENTS 100 } 101 102 if (defined(invoker.constraints)) { 103 _constraints = invoker.constraints 104 } else { 105 _constraints = pw_build_PIP_CONSTRAINTS 106 } 107 108 _python_interpreter = _path + "/bin/python" 109 if (host_os == "win") { 110 _python_interpreter = _path + "/Scripts/python.exe" 111 } 112 113 _venv_metadata_json_file = "$target_gen_dir/$target_name/venv_metadata.json" 114 _venv_metadata = { 115 gn_target_name = 116 get_label_info(":${invoker.target_name}", "label_no_toolchain") 117 path = rebase_path(_path, root_build_dir) 118 generated_requirements = 119 rebase_path(_generated_requirements_file, root_build_dir) 120 compiled_requirements = 121 rebase_path(_compiled_requirements_file, root_build_dir) 122 requirements = rebase_path(_requirements, root_build_dir) 123 constraints = rebase_path(_constraints, root_build_dir) 124 interpreter = rebase_path(_python_interpreter, root_build_dir) 125 source_packages = _source_package_labels 126 } 127 write_file(_venv_metadata_json_file, _venv_metadata, "json") 128 129 pw_python_action("${target_name}._create_virtualenv") { 130 _pw_internal_run_in_venv = false 131 132 # Note: The if the venv isn't in the out dir then we can't declare 133 # outputs and must stamp instead. 134 stamp = true 135 136 # The virtualenv should depend on the version of Python currently in use. 137 stampfile = "$target_gen_dir/$target_name.pw_pystamp" 138 depfile = "$target_gen_dir/$target_name.d" 139 script = "$dir_pw_build/py/pw_build/create_gn_venv.py" 140 args = [ 141 "--depfile", 142 rebase_path(depfile, root_build_dir), 143 "--destination-dir", 144 rebase_path(_path, root_build_dir), 145 "--stampfile", 146 rebase_path(stampfile, root_build_dir), 147 ] 148 } 149 150 # If the venv includes source packages then their 3p install requirements 151 # (defined in setup.cfg files) are collected into a single requirements.txt 152 # file. 153 if (_source_packages != [] && 154 current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 155 pw_python_action("${target_name}._generate_3p_requirements") { 156 inputs = _requirements + _constraints 157 158 _pw_internal_run_in_venv = false 159 _forward_python_metadata_deps = true 160 161 script = "$dir_pw_build/py/pw_build/generate_python_requirements.py" 162 163 _pkg_gn_labels = [] 164 foreach(pkg, _source_packages) { 165 _pkg_gn_labels += [ get_label_info(pkg, "label_no_toolchain") + 166 "($pw_build_PYTHON_TOOLCHAIN)" ] 167 } 168 169 # Add target packages to the python_metadata_deps. This will let 170 # GN expand the transitive pw_python_package deps which are read 171 # by generate_python_requirements.py 172 python_metadata_deps = _pkg_gn_labels 173 174 args = [ 175 "--gn-root-build-dir", 176 rebase_path(root_build_dir, root_build_dir), 177 "--output-requirement-file", 178 rebase_path(_generated_requirements_file, root_build_dir), 179 ] 180 181 if (_constraints != []) { 182 args += [ "--constraint-files" ] 183 } 184 foreach(_constraints_file, _constraints) { 185 args += [ rebase_path(_constraints_file, root_build_dir) ] 186 } 187 188 args += [ 189 "--gn-packages", 190 string_join(",", _pkg_gn_labels), 191 ] 192 193 outputs = [ _generated_requirements_file ] 194 deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 195 } 196 # End _source_packages != [] check 197 } else { 198 # No source packages specified for this venv. Skip collecting 3p install 199 # requirements. 200 group("${target_name}._generate_3p_requirements") { 201 } 202 } 203 204 _pip_generate_hashes = false 205 if (defined(invoker.pip_generate_hashes)) { 206 _pip_generate_hashes = invoker.pip_generate_hashes 207 } else { 208 not_needed([ "_pip_generate_hashes" ]) 209 } 210 211 # Use pip-compile to generate a fully expanded requirements.txt file for all 212 # 3p Python packages. 213 if (_source_packages != [] || _requirements != []) { 214 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 215 # Compile requirements with hashes 216 pw_python_action("${target_name}._compile_requirements") { 217 module = "piptools" 218 219 # Set the venv to run this pip install in. 220 _pw_internal_run_in_venv = true 221 _skip_installing_external_python_deps = true 222 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 223 224 _pip_args = [] 225 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 226 _pip_args += [ "--no-index" ] 227 } 228 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 229 _pip_args += [ "--no-cache-dir" ] 230 } 231 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 232 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 233 _pip_args += 234 [ "--find-links=" + rebase_path(link_dir, root_build_dir) ] 235 } 236 } 237 238 args = [ "compile" ] 239 240 if (_pip_generate_hashes) { 241 args += [ 242 "--generate-hashes", 243 "--reuse-hashes", 244 ] 245 } 246 247 args += [ 248 "--resolver=backtracking", 249 250 # --allow-unsafe will force pinning pip and setuptools. 251 "--allow-unsafe", 252 "--output-file", 253 rebase_path(_compiled_requirements_file, root_build_dir), 254 255 # Input requirements file: 256 rebase_path(_generated_requirements_file, root_build_dir), 257 ] 258 259 # Pass offline related pip args through the pip-compile command. 260 if (_pip_args != []) { 261 args += [ 262 "--pip-args", 263 string_join(" ", _pip_args), 264 ] 265 } 266 267 # Extra requirements files 268 foreach(_requirements_file, _requirements) { 269 args += [ rebase_path(_requirements_file, root_build_dir) ] 270 } 271 272 inputs = [] 273 274 # NOTE: constraint files are included in the content of the 275 # _generated_requirements_file. This occurs in the 276 # ._generate_3p_requirements target. 277 inputs += _constraints 278 inputs += _requirements 279 inputs += [ _generated_requirements_file ] 280 281 deps = [ 282 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 283 ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", 284 ] 285 outputs = [ _compiled_requirements_file ] 286 } 287 288 # This target will run 'pip install' in the venv to provide base 289 # dependencies needed to run piptools commands. That is required for the 290 # _compile_requirements sub target. 291 pw_python_action("${target_name}._install_base_3p_deps") { 292 module = "pip" 293 294 # Set the venv to run this pip install in. 295 _pw_internal_run_in_venv = true 296 _skip_installing_external_python_deps = true 297 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 298 299 _base_requirement_file = "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/python_base_requirements.txt" 300 301 args = [ 302 "install", 303 "--requirement", 304 rebase_path(_base_requirement_file, root_build_dir), 305 ] 306 if (_output_logs) { 307 _pip_install_log_file = 308 "$target_gen_dir/$target_name/pip_install_log.txt" 309 args += [ 310 "--log", 311 rebase_path(_pip_install_log_file, root_build_dir), 312 ] 313 outputs = [ _pip_install_log_file ] 314 } 315 316 # NOTE: Constraints should be ignored for this step. 317 318 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 319 args += [ "--no-index" ] 320 } 321 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 322 args += [ "--no-cache-dir" ] 323 } 324 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 325 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 326 args += [ 327 "--find-links", 328 rebase_path(link_dir, root_build_dir), 329 ] 330 } 331 } 332 333 deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 334 stamp = true 335 pool = "$dir_pw_build/pool:pip($default_toolchain)" 336 } 337 338 # Install all 3rd party Python dependencies. Note the venv 339 # _install_3p_deps sub target is what all pw_python_action targets depend 340 # on by default. 341 pw_python_action("${target_name}._install_3p_deps") { 342 module = "pip" 343 344 # Set the venv to run this pip install in. 345 _pw_internal_run_in_venv = true 346 _skip_installing_external_python_deps = true 347 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 348 349 args = pw_build_PYTHON_PIP_DEFAULT_OPTIONS 350 args += [ 351 "install", 352 "--upgrade", 353 ] 354 355 if (_output_logs) { 356 _pip_install_log_file = 357 "$target_gen_dir/$target_name/pip_install_log.txt" 358 args += [ 359 "--log", 360 rebase_path(_pip_install_log_file, root_build_dir), 361 ] 362 } 363 364 if (_pip_generate_hashes) { 365 args += [ "--require-hashes" ] 366 } 367 368 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 369 args += [ "--no-index" ] 370 } 371 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 372 args += [ "--no-cache-dir" ] 373 } 374 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 375 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 376 args += [ 377 "--find-links", 378 rebase_path(link_dir, root_build_dir), 379 ] 380 } 381 } 382 383 # Note: --no-build-isolation should be avoided for installing 3rd party 384 # Python packages that use C/C++ extension modules. 385 # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html 386 inputs = _constraints + _requirements + [ _compiled_requirements_file ] 387 388 # Use the pip-tools compiled requiremets file. This contains the fully 389 # expanded list of deps with constraints applied. 390 if (defined(invoker.source_packages)) { 391 inputs += [ _compiled_requirements_file ] 392 args += [ 393 "--requirement", 394 rebase_path(_compiled_requirements_file, root_build_dir), 395 ] 396 } 397 398 deps = [ 399 ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", 400 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 401 ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", 402 ] 403 stamp = true 404 pool = "$dir_pw_build/pool:pip($default_toolchain)" 405 } 406 407 # Target to create a Python package cache for this pw_python_venv. 408 pw_python_action("${target_name}.vendor_wheels") { 409 _wheel_output_dir = "$target_gen_dir/$target_name/wheels" 410 _pip_download_logfile = 411 "$target_gen_dir/$target_name/pip_download_log.txt" 412 _pip_wheel_logfile = "$target_gen_dir/$target_name/pip_wheel_log.txt" 413 metadata = { 414 pw_python_package_wheels = [ _wheel_output_dir ] 415 } 416 417 script = "$dir_pw_build/py/pw_build/generate_python_wheel_cache.py" 418 419 # Set the venv to run this pip install in. 420 _pw_internal_run_in_venv = true 421 _skip_installing_external_python_deps = true 422 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 423 424 args = [ 425 "--pip-download-log", 426 rebase_path(_pip_download_logfile, root_build_dir), 427 "--wheel-dir", 428 rebase_path(_wheel_output_dir, root_build_dir), 429 "-r", 430 rebase_path(_compiled_requirements_file, root_build_dir), 431 ] 432 433 if (pw_build_PYTHON_PIP_DOWNLOAD_ALL_PLATFORMS) { 434 args += [ "--download-all-platforms" ] 435 } 436 437 deps = [ 438 ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", 439 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 440 ] 441 442 outputs = [ 443 _wheel_output_dir, 444 _pip_wheel_logfile, 445 _pip_download_logfile, 446 ] 447 pool = "$dir_pw_build/pool:pip($default_toolchain)" 448 } 449 450 # End pw_build_PYTHON_TOOLCHAIN check 451 } else { 452 # Outside the Python toolchain, add deps for the same targets under the 453 # correct toolchain. 454 group("${target_name}._compile_requirements") { 455 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 456 } 457 group("${target_name}._install_3p_deps") { 458 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 459 } 460 group("${target_name}.vendor_wheels") { 461 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 462 } 463 } 464 # End _source_packages != [] || _requirements != [] check 465 } else { 466 group("${target_name}._compile_requirements") { 467 } 468 469 # Note the venv ._install_3p_deps sub target is what all pw_python_action 470 # targets depend on by default. 471 group("${target_name}._install_3p_deps") { 472 # _install_3p_deps should still depend on _create_virtualenv when no 473 # requirements or source packages are provided. 474 public_deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 475 } 476 group("${target_name}.vendor_wheels") { 477 } 478 } 479 480 # Have this target directly depend on _install_3p_deps 481 group("$target_name") { 482 public_deps = 483 [ ":${target_name}._install_3p_deps($pw_build_PYTHON_TOOLCHAIN)" ] 484 } 485} 486