1# Copyright 2024 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Tests for precompiling behavior.""" 16 17load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") 18load("@rules_testing//lib:analysis_test.bzl", "analysis_test") 19load("@rules_testing//lib:test_suite.bzl", "test_suite") 20load("@rules_testing//lib:truth.bzl", "matching") 21load("@rules_testing//lib:util.bzl", rt_util = "util") 22load("//python:py_binary.bzl", "py_binary") 23load("//python:py_info.bzl", "PyInfo") 24load("//python:py_library.bzl", "py_library") 25load("//python:py_test.bzl", "py_test") 26load("//tests/support:py_info_subject.bzl", "py_info_subject") 27load( 28 "//tests/support:support.bzl", 29 "ADD_SRCS_TO_RUNFILES", 30 "CC_TOOLCHAIN", 31 "EXEC_TOOLS_TOOLCHAIN", 32 "PRECOMPILE", 33 "PY_TOOLCHAINS", 34) 35 36_COMMON_CONFIG_SETTINGS = { 37 # This isn't enabled in all environments the tests run in, so disable 38 # it for conformity. 39 "//command_line_option:allow_unresolved_symlinks": True, 40 "//command_line_option:extra_toolchains": [PY_TOOLCHAINS, CC_TOOLCHAIN], 41 EXEC_TOOLS_TOOLCHAIN: "enabled", 42} 43 44_tests = [] 45 46def _test_executable_precompile_attr_enabled_setup(name, py_rule, **kwargs): 47 if not rp_config.enable_pystar: 48 rt_util.skip_test(name = name) 49 return 50 rt_util.helper_target( 51 py_rule, 52 name = name + "_subject", 53 precompile = "enabled", 54 srcs = ["main.py"], 55 deps = [name + "_lib1"], 56 **kwargs 57 ) 58 rt_util.helper_target( 59 py_library, 60 name = name + "_lib1", 61 srcs = ["lib1.py"], 62 precompile = "enabled", 63 deps = [name + "_lib2"], 64 ) 65 66 # 2nd order target to verify propagation 67 rt_util.helper_target( 68 py_library, 69 name = name + "_lib2", 70 srcs = ["lib2.py"], 71 precompile = "enabled", 72 ) 73 analysis_test( 74 name = name, 75 impl = _test_executable_precompile_attr_enabled_impl, 76 target = name + "_subject", 77 config_settings = _COMMON_CONFIG_SETTINGS, 78 ) 79 80def _test_executable_precompile_attr_enabled_impl(env, target): 81 target = env.expect.that_target(target) 82 runfiles = target.runfiles() 83 runfiles_contains_at_least_predicates(runfiles, [ 84 matching.str_matches("__pycache__/main.fakepy-45.pyc"), 85 matching.str_matches("__pycache__/lib1.fakepy-45.pyc"), 86 matching.str_matches("__pycache__/lib2.fakepy-45.pyc"), 87 matching.str_matches("/main.py"), 88 matching.str_matches("/lib1.py"), 89 matching.str_matches("/lib2.py"), 90 ]) 91 92 target.default_outputs().contains_at_least_predicates([ 93 matching.file_path_matches("__pycache__/main.fakepy-45.pyc"), 94 matching.file_path_matches("/main.py"), 95 ]) 96 py_info = target.provider(PyInfo, factory = py_info_subject) 97 py_info.direct_pyc_files().contains_exactly([ 98 "{package}/__pycache__/main.fakepy-45.pyc", 99 ]) 100 py_info.transitive_pyc_files().contains_exactly([ 101 "{package}/__pycache__/main.fakepy-45.pyc", 102 "{package}/__pycache__/lib1.fakepy-45.pyc", 103 "{package}/__pycache__/lib2.fakepy-45.pyc", 104 ]) 105 106def _test_precompile_enabled_py_binary(name): 107 _test_executable_precompile_attr_enabled_setup(name = name, py_rule = py_binary, main = "main.py") 108 109_tests.append(_test_precompile_enabled_py_binary) 110 111def _test_precompile_enabled_py_test(name): 112 _test_executable_precompile_attr_enabled_setup(name = name, py_rule = py_test, main = "main.py") 113 114_tests.append(_test_precompile_enabled_py_test) 115 116def _test_precompile_enabled_py_library_setup(name, impl, config_settings): 117 if not rp_config.enable_pystar: 118 rt_util.skip_test(name = name) 119 return 120 rt_util.helper_target( 121 py_library, 122 name = name + "_subject", 123 srcs = ["lib.py"], 124 precompile = "enabled", 125 ) 126 analysis_test( 127 name = name, 128 impl = impl, #_test_precompile_enabled_py_library_impl, 129 target = name + "_subject", 130 config_settings = _COMMON_CONFIG_SETTINGS | config_settings, 131 ) 132 133def _test_precompile_enabled_py_library_common_impl(env, target): 134 target = env.expect.that_target(target) 135 136 target.default_outputs().contains_at_least_predicates([ 137 matching.file_path_matches("__pycache__/lib.fakepy-45.pyc"), 138 matching.file_path_matches("/lib.py"), 139 ]) 140 py_info = target.provider(PyInfo, factory = py_info_subject) 141 py_info.direct_pyc_files().contains_exactly([ 142 "{package}/__pycache__/lib.fakepy-45.pyc", 143 ]) 144 py_info.transitive_pyc_files().contains_exactly([ 145 "{package}/__pycache__/lib.fakepy-45.pyc", 146 ]) 147 148def _test_precompile_enabled_py_library_add_to_runfiles_disabled(name): 149 _test_precompile_enabled_py_library_setup( 150 name = name, 151 impl = _test_precompile_enabled_py_library_add_to_runfiles_disabled_impl, 152 config_settings = { 153 ADD_SRCS_TO_RUNFILES: "disabled", 154 }, 155 ) 156 157def _test_precompile_enabled_py_library_add_to_runfiles_disabled_impl(env, target): 158 _test_precompile_enabled_py_library_common_impl(env, target) 159 runfiles = env.expect.that_target(target).runfiles() 160 runfiles.contains_exactly([]) 161 162_tests.append(_test_precompile_enabled_py_library_add_to_runfiles_disabled) 163 164def _test_precompile_enabled_py_library_add_to_runfiles_enabled(name): 165 _test_precompile_enabled_py_library_setup( 166 name = name, 167 impl = _test_precompile_enabled_py_library_add_to_runfiles_enabled_impl, 168 config_settings = { 169 ADD_SRCS_TO_RUNFILES: "enabled", 170 }, 171 ) 172 173def _test_precompile_enabled_py_library_add_to_runfiles_enabled_impl(env, target): 174 _test_precompile_enabled_py_library_common_impl(env, target) 175 runfiles = env.expect.that_target(target).runfiles() 176 runfiles.contains_exactly([ 177 "{workspace}/{package}/lib.py", 178 ]) 179 180_tests.append(_test_precompile_enabled_py_library_add_to_runfiles_enabled) 181 182def _test_pyc_only(name): 183 if not rp_config.enable_pystar: 184 rt_util.skip_test(name = name) 185 return 186 rt_util.helper_target( 187 py_binary, 188 name = name + "_subject", 189 precompile = "enabled", 190 srcs = ["main.py"], 191 main = "main.py", 192 precompile_source_retention = "omit_source", 193 pyc_collection = "include_pyc", 194 deps = [name + "_lib"], 195 ) 196 rt_util.helper_target( 197 py_library, 198 name = name + "_lib", 199 srcs = ["lib.py"], 200 precompile_source_retention = "omit_source", 201 ) 202 analysis_test( 203 name = name, 204 impl = _test_pyc_only_impl, 205 config_settings = _COMMON_CONFIG_SETTINGS | { 206 PRECOMPILE: "enabled", 207 }, 208 target = name + "_subject", 209 ) 210 211_tests.append(_test_pyc_only) 212 213def _test_pyc_only_impl(env, target): 214 target = env.expect.that_target(target) 215 runfiles = target.runfiles() 216 runfiles.contains_predicate( 217 matching.str_matches("/main.pyc"), 218 ) 219 runfiles.contains_predicate( 220 matching.str_matches("/lib.pyc"), 221 ) 222 runfiles.not_contains_predicate( 223 matching.str_endswith("/main.py"), 224 ) 225 runfiles.not_contains_predicate( 226 matching.str_endswith("/lib.py"), 227 ) 228 target.default_outputs().contains_at_least_predicates([ 229 matching.file_path_matches("/main.pyc"), 230 ]) 231 target.default_outputs().not_contains_predicate( 232 matching.file_basename_equals("main.py"), 233 ) 234 235def _test_precompiler_action(name): 236 if not rp_config.enable_pystar: 237 rt_util.skip_test(name = name) 238 return 239 rt_util.helper_target( 240 py_binary, 241 name = name + "_subject", 242 srcs = ["main2.py"], 243 main = "main2.py", 244 precompile = "enabled", 245 precompile_optimize_level = 2, 246 precompile_invalidation_mode = "unchecked_hash", 247 ) 248 analysis_test( 249 name = name, 250 impl = _test_precompiler_action_impl, 251 target = name + "_subject", 252 config_settings = _COMMON_CONFIG_SETTINGS, 253 ) 254 255_tests.append(_test_precompiler_action) 256 257def _test_precompiler_action_impl(env, target): 258 action = env.expect.that_target(target).action_named("PyCompile") 259 action.contains_flag_values([ 260 ("--optimize", "2"), 261 ("--python_version", "4.5"), 262 ("--invalidation_mode", "unchecked_hash"), 263 ]) 264 action.has_flags_specified(["--src", "--pyc", "--src_name"]) 265 action.env().contains_at_least({ 266 "PYTHONHASHSEED": "0", 267 "PYTHONNOUSERSITE": "1", 268 "PYTHONSAFEPATH": "1", 269 }) 270 271def _setup_precompile_flag_pyc_collection_attr_interaction( 272 *, 273 name, 274 pyc_collection_attr, 275 precompile_flag, 276 test_impl): 277 rt_util.helper_target( 278 py_binary, 279 name = name + "_bin", 280 srcs = ["bin.py"], 281 main = "bin.py", 282 precompile = "disabled", 283 pyc_collection = pyc_collection_attr, 284 deps = [ 285 name + "_lib_inherit", 286 name + "_lib_enabled", 287 name + "_lib_disabled", 288 ], 289 ) 290 rt_util.helper_target( 291 py_library, 292 name = name + "_lib_inherit", 293 srcs = ["lib_inherit.py"], 294 precompile = "inherit", 295 ) 296 rt_util.helper_target( 297 py_library, 298 name = name + "_lib_enabled", 299 srcs = ["lib_enabled.py"], 300 precompile = "enabled", 301 ) 302 rt_util.helper_target( 303 py_library, 304 name = name + "_lib_disabled", 305 srcs = ["lib_disabled.py"], 306 precompile = "disabled", 307 ) 308 analysis_test( 309 name = name, 310 impl = test_impl, 311 target = name + "_bin", 312 config_settings = _COMMON_CONFIG_SETTINGS | { 313 PRECOMPILE: precompile_flag, 314 }, 315 ) 316 317def _verify_runfiles(contains_patterns, not_contains_patterns): 318 def _verify_runfiles_impl(env, target): 319 runfiles = env.expect.that_target(target).runfiles() 320 for pattern in contains_patterns: 321 runfiles.contains_predicate(matching.str_matches(pattern)) 322 for pattern in not_contains_patterns: 323 runfiles.not_contains_predicate( 324 matching.str_matches(pattern), 325 ) 326 327 return _verify_runfiles_impl 328 329def _test_precompile_flag_enabled_pyc_collection_attr_include_pyc(name): 330 if not rp_config.enable_pystar: 331 rt_util.skip_test(name = name) 332 return 333 _setup_precompile_flag_pyc_collection_attr_interaction( 334 name = name, 335 precompile_flag = "enabled", 336 pyc_collection_attr = "include_pyc", 337 test_impl = _verify_runfiles( 338 contains_patterns = [ 339 "__pycache__/lib_enabled.*.pyc", 340 "__pycache__/lib_inherit.*.pyc", 341 ], 342 not_contains_patterns = [ 343 "/bin*.pyc", 344 "/lib_disabled*.pyc", 345 ], 346 ), 347 ) 348 349_tests.append(_test_precompile_flag_enabled_pyc_collection_attr_include_pyc) 350 351# buildifier: disable=function-docstring-header 352def _test_precompile_flag_enabled_pyc_collection_attr_disabled(name): 353 """Verify that a binary can opt-out of using implicit pycs even when 354 precompiling is enabled by default. 355 """ 356 if not rp_config.enable_pystar: 357 rt_util.skip_test(name = name) 358 return 359 _setup_precompile_flag_pyc_collection_attr_interaction( 360 name = name, 361 precompile_flag = "enabled", 362 pyc_collection_attr = "disabled", 363 test_impl = _verify_runfiles( 364 contains_patterns = [ 365 "__pycache__/lib_enabled.*.pyc", 366 ], 367 not_contains_patterns = [ 368 "/bin*.pyc", 369 "/lib_disabled*.pyc", 370 "/lib_inherit.*.pyc", 371 ], 372 ), 373 ) 374 375_tests.append(_test_precompile_flag_enabled_pyc_collection_attr_disabled) 376 377# buildifier: disable=function-docstring-header 378def _test_precompile_flag_disabled_pyc_collection_attr_include_pyc(name): 379 """Verify that a binary can opt-in to using pycs even when precompiling is 380 disabled by default.""" 381 if not rp_config.enable_pystar: 382 rt_util.skip_test(name = name) 383 return 384 _setup_precompile_flag_pyc_collection_attr_interaction( 385 name = name, 386 precompile_flag = "disabled", 387 pyc_collection_attr = "include_pyc", 388 test_impl = _verify_runfiles( 389 contains_patterns = [ 390 "__pycache__/lib_enabled.*.pyc", 391 "__pycache__/lib_inherit.*.pyc", 392 ], 393 not_contains_patterns = [ 394 "/bin*.pyc", 395 "/lib_disabled*.pyc", 396 ], 397 ), 398 ) 399 400_tests.append(_test_precompile_flag_disabled_pyc_collection_attr_include_pyc) 401 402def _test_precompile_flag_disabled_pyc_collection_attr_disabled(name): 403 if not rp_config.enable_pystar: 404 rt_util.skip_test(name = name) 405 return 406 _setup_precompile_flag_pyc_collection_attr_interaction( 407 name = name, 408 precompile_flag = "disabled", 409 pyc_collection_attr = "disabled", 410 test_impl = _verify_runfiles( 411 contains_patterns = [ 412 "__pycache__/lib_enabled.*.pyc", 413 ], 414 not_contains_patterns = [ 415 "/bin*.pyc", 416 "/lib_disabled*.pyc", 417 "/lib_inherit.*.pyc", 418 ], 419 ), 420 ) 421 422_tests.append(_test_precompile_flag_disabled_pyc_collection_attr_disabled) 423 424# buildifier: disable=function-docstring-header 425def _test_pyc_collection_disabled_library_omit_source(name): 426 """Verify that, when a binary doesn't include implicit pyc files, libraries 427 that set omit_source still have the py source file included. 428 """ 429 if not rp_config.enable_pystar: 430 rt_util.skip_test(name = name) 431 return 432 rt_util.helper_target( 433 py_binary, 434 name = name + "_subject", 435 srcs = ["bin.py"], 436 main = "bin.py", 437 deps = [name + "_lib"], 438 pyc_collection = "disabled", 439 ) 440 rt_util.helper_target( 441 py_library, 442 name = name + "_lib", 443 srcs = ["lib.py"], 444 precompile = "inherit", 445 precompile_source_retention = "omit_source", 446 ) 447 analysis_test( 448 name = name, 449 impl = _test_pyc_collection_disabled_library_omit_source_impl, 450 target = name + "_subject", 451 config_settings = _COMMON_CONFIG_SETTINGS, 452 ) 453 454def _test_pyc_collection_disabled_library_omit_source_impl(env, target): 455 contains_patterns = [ 456 "/lib.py", 457 "/bin.py", 458 ] 459 not_contains_patterns = [ 460 "/lib.*pyc", 461 "/bin.*pyc", 462 ] 463 runfiles = env.expect.that_target(target).runfiles() 464 for pattern in contains_patterns: 465 runfiles.contains_predicate(matching.str_matches(pattern)) 466 for pattern in not_contains_patterns: 467 runfiles.not_contains_predicate( 468 matching.str_matches(pattern), 469 ) 470 471_tests.append(_test_pyc_collection_disabled_library_omit_source) 472 473def _test_pyc_collection_include_dep_omit_source(name): 474 if not rp_config.enable_pystar: 475 rt_util.skip_test(name = name) 476 return 477 rt_util.helper_target( 478 py_binary, 479 name = name + "_subject", 480 srcs = ["bin.py"], 481 main = "bin.py", 482 deps = [name + "_lib"], 483 precompile = "disabled", 484 pyc_collection = "include_pyc", 485 ) 486 rt_util.helper_target( 487 py_library, 488 name = name + "_lib", 489 srcs = ["lib.py"], 490 precompile = "inherit", 491 precompile_source_retention = "omit_source", 492 ) 493 analysis_test( 494 name = name, 495 impl = _test_pyc_collection_include_dep_omit_source_impl, 496 target = name + "_subject", 497 config_settings = _COMMON_CONFIG_SETTINGS, 498 ) 499 500def _test_pyc_collection_include_dep_omit_source_impl(env, target): 501 contains_patterns = [ 502 "/lib.pyc", 503 ] 504 not_contains_patterns = [ 505 "/lib.py", 506 ] 507 runfiles = env.expect.that_target(target).runfiles() 508 for pattern in contains_patterns: 509 runfiles.contains_predicate(matching.str_endswith(pattern)) 510 for pattern in not_contains_patterns: 511 runfiles.not_contains_predicate( 512 matching.str_endswith(pattern), 513 ) 514 515_tests.append(_test_pyc_collection_include_dep_omit_source) 516 517def _test_precompile_attr_inherit_pyc_collection_disabled_precompile_flag_enabled(name): 518 if not rp_config.enable_pystar: 519 rt_util.skip_test(name = name) 520 return 521 rt_util.helper_target( 522 py_binary, 523 name = name + "_subject", 524 srcs = ["bin.py"], 525 main = "bin.py", 526 precompile = "inherit", 527 pyc_collection = "disabled", 528 ) 529 analysis_test( 530 name = name, 531 impl = _test_precompile_attr_inherit_pyc_collection_disabled_precompile_flag_enabled_impl, 532 target = name + "_subject", 533 config_settings = _COMMON_CONFIG_SETTINGS | { 534 PRECOMPILE: "enabled", 535 }, 536 ) 537 538def _test_precompile_attr_inherit_pyc_collection_disabled_precompile_flag_enabled_impl(env, target): 539 target = env.expect.that_target(target) 540 target.runfiles().not_contains_predicate( 541 matching.str_matches("/bin.*pyc"), 542 ) 543 target.default_outputs().not_contains_predicate( 544 matching.file_path_matches("/bin.*pyc"), 545 ) 546 547_tests.append(_test_precompile_attr_inherit_pyc_collection_disabled_precompile_flag_enabled) 548 549def runfiles_contains_at_least_predicates(runfiles, predicates): 550 for predicate in predicates: 551 runfiles.contains_predicate(predicate) 552 553def precompile_test_suite(name): 554 test_suite( 555 name = name, 556 tests = _tests, 557 ) 558