1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15import("//build_overrides/pigweed.gni") 16 17import("$dir_pw_build/python.gni") 18import("$dir_pw_build/python_action.gni") 19 20# Defines and creates a Python virtualenv. This template is used by Pigweed in 21# https://cs.pigweed.dev/pigweed/+/main:pw_env_setup/BUILD.gn to 22# create a virtualenv for use within the GN build that all Python actions will 23# run in. 24# 25# Example: 26# 27# pw_python_venv("test_venv") { 28# path = "test-venv" 29# constraints = [ "//tools/constraints.list" ] 30# requirements = [ "//tools/requirements.txt" ] 31# source_packages = [ 32# "$dir_pw_cli/py", 33# "$dir_pw_console/py", 34# "//tools:another_pw_python_package", 35# ] 36# } 37# 38# Args: 39# path: The directory where the virtualenv will be created. This is relative 40# to the GN root and must begin with "$root_build_dir/" if it lives in the 41# output directory or "//" if it lives in elsewhere. 42# 43# constraints: A list of constraint files used when performing pip install 44# into this virtualenv. By default this is set to pw_build_PIP_CONSTRAINTS 45# 46# requirements: A list of requirements files to install into this virtualenv 47# on creation. By default this is set to pw_build_PIP_REQUIREMENTS 48# 49# pip_generate_hashes: (Default: false) Use --generate-hashes When 50# running pip-compile to compute the final requirements.txt 51# 52# source_packages: A list of in-tree pw_python_package targets that will be 53# checked for external third_party pip dependencies to install into this 54# virtualenv. Note this list of targets isn't actually installed into the 55# virtualenv. Only packages defined inside the [options] install_requires 56# section of each pw_python_package's setup.cfg will be pip installed. See 57# this page for a setup.cfg example: 58# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html 59# 60# output_logs: (Default: true) Commands will output logs. 61# 62template("pw_python_venv") { 63 assert(defined(invoker.path), "pw_python_venv requires a 'path'") 64 65 _path = invoker.path 66 67 _generated_requirements_file = 68 "$target_gen_dir/$target_name/generated_requirements.txt" 69 70 _compiled_requirements_file = 71 "$target_gen_dir/$target_name/compiled_requirements.txt" 72 73 _source_packages = [] 74 if (defined(invoker.source_packages)) { 75 _source_packages += invoker.source_packages 76 } else { 77 not_needed([ 78 "_source_packages", 79 "_generated_requirements_file", 80 ]) 81 } 82 _output_logs = true 83 if (defined(invoker.output_logs)) { 84 _output_logs = invoker.output_logs 85 } 86 if (!defined(invoker.output_logs) || 87 current_toolchain != pw_build_PYTHON_TOOLCHAIN) { 88 not_needed([ "_output_logs" ]) 89 } 90 91 _source_package_labels = [] 92 foreach(pkg, _source_packages) { 93 _source_package_labels += [ get_label_info(pkg, "label_no_toolchain") ] 94 } 95 96 if (defined(invoker.requirements)) { 97 _requirements = invoker.requirements 98 } else { 99 _requirements = pw_build_PIP_REQUIREMENTS 100 } 101 102 if (defined(invoker.constraints)) { 103 _constraints = invoker.constraints 104 } else { 105 _constraints = pw_build_PIP_CONSTRAINTS 106 } 107 108 _python_interpreter = _path + "/bin/python" 109 if (host_os == "win") { 110 _python_interpreter = _path + "/Scripts/python.exe" 111 } 112 113 _venv_metadata_json_file = "$target_gen_dir/$target_name/venv_metadata.json" 114 _venv_metadata = { 115 gn_target_name = 116 get_label_info(":${invoker.target_name}", "label_no_toolchain") 117 path = rebase_path(_path, root_build_dir) 118 generated_requirements = 119 rebase_path(_generated_requirements_file, root_build_dir) 120 compiled_requirements = 121 rebase_path(_compiled_requirements_file, root_build_dir) 122 requirements = rebase_path(_requirements, root_build_dir) 123 constraints = rebase_path(_constraints, root_build_dir) 124 interpreter = rebase_path(_python_interpreter, root_build_dir) 125 source_packages = _source_package_labels 126 } 127 write_file(_venv_metadata_json_file, _venv_metadata, "json") 128 129 pw_python_action("${target_name}._create_virtualenv") { 130 _pw_internal_run_in_venv = false 131 132 # Note: The if the venv isn't in the out dir then we can't declare 133 # outputs and must stamp instead. 134 stamp = true 135 136 # The virtualenv should depend on the version of Python currently in use. 137 stampfile = "$target_gen_dir/$target_name.pw_pystamp" 138 depfile = "$target_gen_dir/$target_name.d" 139 script = "$dir_pw_build/py/pw_build/create_gn_venv.py" 140 args = [ 141 "--depfile", 142 rebase_path(depfile, root_build_dir), 143 "--destination-dir", 144 rebase_path(_path, root_build_dir), 145 "--stampfile", 146 rebase_path(stampfile, root_build_dir), 147 ] 148 } 149 150 if (defined(invoker.source_packages) && 151 current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 152 pw_python_action("${target_name}._generate_3p_requirements") { 153 inputs = _requirements + _constraints 154 155 _pw_internal_run_in_venv = false 156 _forward_python_metadata_deps = true 157 158 script = "$dir_pw_build/py/pw_build/generate_python_requirements.py" 159 160 _pkg_gn_labels = [] 161 foreach(pkg, _source_packages) { 162 _pkg_gn_labels += [ get_label_info(pkg, "label_no_toolchain") + 163 "($pw_build_PYTHON_TOOLCHAIN)" ] 164 } 165 166 # pw_build/py is always needed for venv creation and Python lint checks. 167 python_metadata_deps = 168 [ get_label_info("$dir_pw_build/py", "label_no_toolchain") + 169 "($pw_build_PYTHON_TOOLCHAIN)" ] 170 python_metadata_deps += _pkg_gn_labels 171 172 args = [ 173 "--gn-root-build-dir", 174 rebase_path(root_build_dir, root_build_dir), 175 "--output-requirement-file", 176 rebase_path(_generated_requirements_file, root_build_dir), 177 ] 178 179 if (_constraints != []) { 180 args += [ "--constraint-files" ] 181 } 182 foreach(_constraints_file, _constraints) { 183 args += [ rebase_path(_constraints_file, root_build_dir) ] 184 } 185 186 args += [ 187 "--gn-packages", 188 string_join(",", _pkg_gn_labels), 189 ] 190 191 outputs = [ _generated_requirements_file ] 192 deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 193 } 194 } else { 195 group("${target_name}._generate_3p_requirements") { 196 } 197 } 198 199 _pip_generate_hashes = false 200 if (defined(invoker.pip_generate_hashes)) { 201 _pip_generate_hashes = invoker.pip_generate_hashes 202 } else { 203 not_needed([ "_pip_generate_hashes" ]) 204 } 205 206 if (defined(invoker.source_packages) || defined(invoker.requirements)) { 207 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) { 208 # Compile requirements with hashes 209 pw_python_action("${target_name}._compile_requirements") { 210 module = "piptools" 211 212 # Set the venv to run this pip install in. 213 _pw_internal_run_in_venv = true 214 _skip_installing_external_python_deps = true 215 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 216 217 _pip_args = [] 218 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 219 _pip_args += [ "--no-index" ] 220 } 221 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 222 _pip_args += [ "--no-cache-dir" ] 223 } 224 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 225 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 226 _pip_args += 227 [ "--find-links=" + rebase_path(link_dir, root_build_dir) ] 228 } 229 } 230 231 args = [ "compile" ] 232 233 if (_pip_generate_hashes) { 234 args += [ 235 "--generate-hashes", 236 "--reuse-hashes", 237 ] 238 } 239 240 args += [ 241 "--resolver=backtracking", 242 243 # --allow-unsafe will force pinning pip and setuptools. 244 "--allow-unsafe", 245 "--output-file", 246 rebase_path(_compiled_requirements_file, root_build_dir), 247 248 # Input requirements file: 249 rebase_path(_generated_requirements_file, root_build_dir), 250 ] 251 252 # Pass offline related pip args through the pip-compile command. 253 if (_pip_args != []) { 254 args += [ 255 "--pip-args", 256 string_join(" ", _pip_args), 257 ] 258 } 259 260 # Extra requirements files 261 foreach(_requirements_file, _requirements) { 262 args += [ rebase_path(_requirements_file, root_build_dir) ] 263 } 264 265 inputs = [] 266 267 # NOTE: constraint files are included in the content of the 268 # _generated_requirements_file. This occurs in the 269 # ._generate_3p_requirements target. 270 inputs += _constraints 271 inputs += _requirements 272 inputs += [ _generated_requirements_file ] 273 274 deps = [ 275 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 276 ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", 277 ] 278 outputs = [ _compiled_requirements_file ] 279 } 280 281 # This target will run 'pip install' in the venv to provide base 282 # dependencies needed to run piptools commands. That is required for the 283 # _compile_requirements sub target. 284 pw_python_action("${target_name}._install_base_3p_deps") { 285 module = "pip" 286 287 # Set the venv to run this pip install in. 288 _pw_internal_run_in_venv = true 289 _skip_installing_external_python_deps = true 290 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 291 292 _base_requirement_file = "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/python_base_requirements.txt" 293 294 args = [ 295 "install", 296 "--requirement", 297 rebase_path(_base_requirement_file, root_build_dir), 298 ] 299 if (_output_logs) { 300 _pip_install_log_file = 301 "$target_gen_dir/$target_name/pip_install_log.txt" 302 args += [ 303 "--log", 304 rebase_path(_pip_install_log_file, root_build_dir), 305 ] 306 outputs = [ _pip_install_log_file ] 307 } 308 309 # NOTE: Constraints should be ignored for this step. 310 311 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 312 args += [ "--no-index" ] 313 } 314 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 315 args += [ "--no-cache-dir" ] 316 } 317 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 318 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 319 args += [ 320 "--find-links", 321 rebase_path(link_dir, root_build_dir), 322 ] 323 } 324 } 325 326 deps = [ ":${invoker.target_name}._create_virtualenv($pw_build_PYTHON_TOOLCHAIN)" ] 327 stamp = true 328 pool = "$dir_pw_build/pool:pip($default_toolchain)" 329 } 330 331 # Install all 3rd party Python dependencies. 332 pw_python_action("${target_name}._install_3p_deps") { 333 module = "pip" 334 335 # Set the venv to run this pip install in. 336 _pw_internal_run_in_venv = true 337 _skip_installing_external_python_deps = true 338 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 339 340 args = pw_build_PYTHON_PIP_DEFAULT_OPTIONS 341 args += [ 342 "install", 343 "--upgrade", 344 ] 345 346 if (_output_logs) { 347 _pip_install_log_file = 348 "$target_gen_dir/$target_name/pip_install_log.txt" 349 args += [ 350 "--log", 351 rebase_path(_pip_install_log_file, root_build_dir), 352 ] 353 } 354 355 if (_pip_generate_hashes) { 356 args += [ "--require-hashes" ] 357 } 358 359 if (pw_build_PYTHON_PIP_INSTALL_OFFLINE) { 360 args += [ "--no-index" ] 361 } 362 if (pw_build_PYTHON_PIP_INSTALL_DISABLE_CACHE) { 363 args += [ "--no-cache-dir" ] 364 } 365 if (pw_build_PYTHON_PIP_INSTALL_FIND_LINKS != []) { 366 foreach(link_dir, pw_build_PYTHON_PIP_INSTALL_FIND_LINKS) { 367 args += [ 368 "--find-links", 369 rebase_path(link_dir, root_build_dir), 370 ] 371 } 372 } 373 374 # Note: --no-build-isolation should be avoided for installing 3rd party 375 # Python packages that use C/C++ extension modules. 376 # https://setuptools.pypa.io/en/latest/userguide/ext_modules.html 377 inputs = _constraints + _requirements + [ _compiled_requirements_file ] 378 379 # Use the pip-tools compiled requiremets file. This contains the fully 380 # expanded list of deps with constraints applied. 381 if (defined(invoker.source_packages)) { 382 inputs += [ _compiled_requirements_file ] 383 args += [ 384 "--requirement", 385 rebase_path(_compiled_requirements_file, root_build_dir), 386 ] 387 } 388 389 deps = [ 390 ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", 391 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 392 ":${invoker.target_name}._install_base_3p_deps($pw_build_PYTHON_TOOLCHAIN)", 393 ] 394 stamp = true 395 pool = "$dir_pw_build/pool:pip($default_toolchain)" 396 } 397 398 # Target to create a Python package cache for this pw_python_venv. 399 pw_python_action("${target_name}.vendor_wheels") { 400 _wheel_output_dir = "$target_gen_dir/$target_name/wheels" 401 _pip_download_logfile = 402 "$target_gen_dir/$target_name/pip_download_log.txt" 403 _pip_wheel_logfile = "$target_gen_dir/$target_name/pip_wheel_log.txt" 404 metadata = { 405 pw_python_package_wheels = [ _wheel_output_dir ] 406 } 407 408 script = "$dir_pw_build/py/pw_build/generate_python_wheel_cache.py" 409 410 # Set the venv to run this pip install in. 411 _pw_internal_run_in_venv = true 412 _skip_installing_external_python_deps = true 413 venv = get_label_info(":${invoker.target_name}", "label_no_toolchain") 414 415 args = [ 416 "--pip-download-log", 417 rebase_path(_pip_download_logfile, root_build_dir), 418 "--wheel-dir", 419 rebase_path(_wheel_output_dir, root_build_dir), 420 "-r", 421 rebase_path(_compiled_requirements_file, root_build_dir), 422 ] 423 424 if (pw_build_PYTHON_PIP_DOWNLOAD_ALL_PLATFORMS) { 425 args += [ "--download-all-platforms" ] 426 } 427 428 deps = [ 429 ":${invoker.target_name}._compile_requirements($pw_build_PYTHON_TOOLCHAIN)", 430 ":${invoker.target_name}._generate_3p_requirements($pw_build_PYTHON_TOOLCHAIN)", 431 ] 432 433 outputs = [ 434 _wheel_output_dir, 435 _pip_wheel_logfile, 436 _pip_download_logfile, 437 ] 438 pool = "$dir_pw_build/pool:pip($default_toolchain)" 439 } 440 441 # End pw_build_PYTHON_TOOLCHAIN check 442 } else { 443 group("${target_name}._compile_requirements") { 444 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 445 } 446 group("${target_name}._install_3p_deps") { 447 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 448 } 449 group("${target_name}.vendor_wheels") { 450 public_deps = [ ":${target_name}($pw_build_PYTHON_TOOLCHAIN)" ] 451 } 452 } 453 } else { 454 group("${target_name}._compile_requirements") { 455 } 456 group("${target_name}._install_3p_deps") { 457 } 458 group("${target_name}.vendor_wheels") { 459 } 460 } 461 462 # Have this target directly depend on _install_3p_deps 463 group("$target_name") { 464 public_deps = 465 [ ":${target_name}._install_3p_deps($pw_build_PYTHON_TOOLCHAIN)" ] 466 } 467} 468