1# Copyright 2021, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Implementation of Atest's Bazel mode. 17 18Bazel mode runs tests using Bazel by generating a synthetic workspace that 19contains test targets. Using Bazel allows Atest to leverage features such as 20sandboxing, caching, and remote execution. 21""" 22# pylint: disable=missing-function-docstring 23# pylint: disable=missing-class-docstring 24# pylint: disable=too-many-lines 25 26from __future__ import annotations 27 28import argparse 29import contextlib 30import dataclasses 31import enum 32import functools 33import logging 34import os 35import re 36import shlex 37import shutil 38import subprocess 39import tempfile 40import time 41import warnings 42 43from abc import ABC, abstractmethod 44from collections import defaultdict, deque, OrderedDict 45from collections.abc import Iterable 46from pathlib import Path 47from types import MappingProxyType 48from typing import Any, Callable, Dict, IO, List, Set 49from xml.etree import ElementTree as ET 50 51from google.protobuf.message import DecodeError 52 53from atest import atest_utils 54from atest import constants 55from atest import module_info 56 57from atest.atest_enum import DetectType, ExitCode 58from atest.metrics import metrics 59from atest.proto import file_md5_pb2 60from atest.test_finders import test_finder_base 61from atest.test_finders import test_info 62from atest.test_runners import test_runner_base as trb 63from atest.test_runners import atest_tf_test_runner as tfr 64 65 66JDK_PACKAGE_NAME = 'prebuilts/robolectric_jdk' 67JDK_NAME = 'jdk' 68ROBOLECTRIC_CONFIG = 'build/make/core/robolectric_test_config_template.xml' 69 70_BAZEL_WORKSPACE_DIR = 'atest_bazel_workspace' 71_SUPPORTED_BAZEL_ARGS = MappingProxyType({ 72 # https://docs.bazel.build/versions/main/command-line-reference.html#flag--runs_per_test 73 constants.ITERATIONS: 74 lambda arg_value: [f'--runs_per_test={str(arg_value)}'], 75 # https://docs.bazel.build/versions/main/command-line-reference.html#flag--test_keep_going 76 constants.RERUN_UNTIL_FAILURE: 77 lambda arg_value: 78 ['--notest_keep_going', f'--runs_per_test={str(arg_value)}'], 79 # https://docs.bazel.build/versions/main/command-line-reference.html#flag--flaky_test_attempts 80 constants.RETRY_ANY_FAILURE: 81 lambda arg_value: [f'--flaky_test_attempts={str(arg_value)}'], 82 # https://docs.bazel.build/versions/main/command-line-reference.html#flag--test_output 83 constants.VERBOSE: 84 lambda arg_value: ['--test_output=all'] if arg_value else [], 85 constants.BAZEL_ARG: 86 lambda arg_value: [item for sublist in arg_value for item in sublist] 87}) 88 89# Maps Bazel configuration names to Soong variant names. 90_CONFIG_TO_VARIANT = { 91 'host': 'host', 92 'device': 'target', 93} 94 95 96class AbortRunException(Exception): 97 pass 98 99 100@enum.unique 101class Features(enum.Enum): 102 NULL_FEATURE = ('--null-feature', 'Enables a no-action feature.', True) 103 EXPERIMENTAL_DEVICE_DRIVEN_TEST = ( 104 '--experimental-device-driven-test', 105 'Enables running device-driven tests in Bazel mode.', True) 106 EXPERIMENTAL_BES_PUBLISH = ('--experimental-bes-publish', 107 'Upload test results via BES in Bazel mode.', 108 False) 109 EXPERIMENTAL_JAVA_RUNTIME_DEPENDENCIES = ( 110 '--experimental-java-runtime-dependencies', 111 'Mirrors Soong Java `libs` and `static_libs` as Bazel target ' 112 'dependencies in the generated workspace. Tradefed test rules use ' 113 'these dependencies to set up the execution environment and ensure ' 114 'that all transitive runtime dependencies are present.', 115 True) 116 EXPERIMENTAL_REMOTE = ( 117 '--experimental-remote', 118 'Use Bazel remote execution and caching where supported.', 119 False) 120 EXPERIMENTAL_HOST_DRIVEN_TEST = ( 121 '--experimental-host-driven-test', 122 'Enables running host-driven device tests in Bazel mode.', True) 123 EXPERIMENTAL_ROBOLECTRIC_TEST = ( 124 '--experimental-robolectric-test', 125 'Enables running Robolectric tests in Bazel mode.', True) 126 127 def __init__(self, arg_flag, description, affects_workspace): 128 self._arg_flag = arg_flag 129 self._description = description 130 self.affects_workspace = affects_workspace 131 132 @property 133 def arg_flag(self): 134 return self._arg_flag 135 136 @property 137 def description(self): 138 return self._description 139 140 141def add_parser_arguments(parser: argparse.ArgumentParser, dest: str): 142 for _, member in Features.__members__.items(): 143 parser.add_argument(member.arg_flag, 144 action='append_const', 145 const=member, 146 dest=dest, 147 help=member.description) 148 149 150def get_bazel_workspace_dir() -> Path: 151 return Path(atest_utils.get_build_out_dir()).joinpath(_BAZEL_WORKSPACE_DIR) 152 153 154def generate_bazel_workspace(mod_info: module_info.ModuleInfo, 155 enabled_features: Set[Features] = None): 156 """Generate or update the Bazel workspace used for running tests.""" 157 158 src_root_path = Path(os.environ.get(constants.ANDROID_BUILD_TOP)) 159 workspace_path = get_bazel_workspace_dir() 160 resource_manager = ResourceManager( 161 src_root_path=src_root_path, 162 resource_root_path=_get_resource_root(), 163 product_out_path=Path( 164 os.environ.get(constants.ANDROID_PRODUCT_OUT)), 165 md5_checksum_file_path=workspace_path.joinpath( 166 'workspace_md5_checksum'), 167 ) 168 jdk_path = _read_robolectric_jdk_path( 169 resource_manager.get_src_file_path(ROBOLECTRIC_CONFIG, True)) 170 171 workspace_generator = WorkspaceGenerator( 172 resource_manager=resource_manager, 173 workspace_out_path=workspace_path, 174 host_out_path=Path(os.environ.get(constants.ANDROID_HOST_OUT)), 175 build_out_dir=Path(atest_utils.get_build_out_dir()), 176 mod_info=mod_info, 177 jdk_path=jdk_path, 178 enabled_features=enabled_features, 179 ) 180 workspace_generator.generate() 181 182 183def get_default_build_metadata(): 184 return BuildMetadata(atest_utils.get_manifest_branch(), 185 atest_utils.get_build_target()) 186 187 188class ResourceManager: 189 """Class for managing files required to generate a Bazel Workspace.""" 190 191 def __init__(self, 192 src_root_path: Path, 193 resource_root_path: Path, 194 product_out_path: Path, 195 md5_checksum_file_path: Path): 196 self._root_type_to_path = { 197 file_md5_pb2.RootType.SRC_ROOT: src_root_path, 198 file_md5_pb2.RootType.RESOURCE_ROOT: resource_root_path, 199 file_md5_pb2.RootType.ABS_PATH: Path(), 200 file_md5_pb2.RootType.PRODUCT_OUT: product_out_path, 201 } 202 self._md5_checksum_file = md5_checksum_file_path 203 self._file_checksum_list = file_md5_pb2.FileChecksumList() 204 205 def get_src_file_path( 206 self, 207 rel_path: Path=None, 208 affects_workspace: bool=False 209 ) -> Path: 210 """Get the abs file path from the relative path of source_root. 211 212 Args: 213 rel_path: A relative path of the source_root. 214 affects_workspace: A boolean of whether the file affects the 215 workspace. 216 217 Returns: 218 A abs path of the file. 219 """ 220 return self._get_file_path( 221 file_md5_pb2.RootType.SRC_ROOT, rel_path, affects_workspace) 222 223 def get_resource_file_path( 224 self, 225 rel_path: Path=None, 226 affects_workspace: bool=False, 227 ) -> Path: 228 """Get the abs file path from the relative path of resource_root. 229 230 Args: 231 rel_path: A relative path of the resource_root. 232 affects_workspace: A boolean of whether the file affects the 233 workspace. 234 235 Returns: 236 A abs path of the file. 237 """ 238 return self._get_file_path( 239 file_md5_pb2.RootType.RESOURCE_ROOT, rel_path, affects_workspace) 240 241 def get_product_out_file_path( 242 self, 243 rel_path: Path=None, 244 affects_workspace: bool=False 245 ) -> Path: 246 """Get the abs file path from the relative path of product out. 247 248 Args: 249 rel_path: A relative path to the product out. 250 affects_workspace: A boolean of whether the file affects the 251 workspace. 252 253 Returns: 254 An abs path of the file. 255 """ 256 return self._get_file_path( 257 file_md5_pb2.RootType.PRODUCT_OUT, rel_path, affects_workspace) 258 259 def _get_file_path( 260 self, 261 root_type: file_md5_pb2.RootType, 262 rel_path: Path, 263 affects_workspace: bool=True 264 ) -> Path: 265 abs_path = self._root_type_to_path[root_type].joinpath( 266 rel_path or Path()) 267 268 if not affects_workspace: 269 return abs_path 270 271 if abs_path.is_dir(): 272 for file in abs_path.glob('**/*'): 273 self._register_file(root_type, file) 274 else: 275 self._register_file(root_type, abs_path) 276 return abs_path 277 278 def _register_file( 279 self, 280 root_type: file_md5_pb2.RootType, 281 abs_path: Path 282 ): 283 if not abs_path.is_file(): 284 logging.debug(' ignore %s: not a file.', abs_path) 285 return 286 287 rel_path = abs_path 288 if abs_path.is_relative_to(self._root_type_to_path[root_type]): 289 rel_path = abs_path.relative_to(self._root_type_to_path[root_type]) 290 291 self._file_checksum_list.file_checksums.append( 292 file_md5_pb2.FileChecksum( 293 root_type=root_type, 294 rel_path=str(rel_path), 295 md5sum=atest_utils.md5sum(abs_path) 296 ) 297 ) 298 299 def register_file_with_abs_path(self, abs_path: Path): 300 """Register a file which affects the workspace. 301 302 Args: 303 abs_path: A abs path of the file. 304 """ 305 self._register_file(file_md5_pb2.RootType.ABS_PATH, abs_path) 306 307 def save_affects_files_md5(self): 308 with open(self._md5_checksum_file, 'wb') as f: 309 f.write(self._file_checksum_list.SerializeToString()) 310 311 def check_affects_files_md5(self): 312 """Check all affect files are consistent with the actual MD5.""" 313 if not self._md5_checksum_file.is_file(): 314 return False 315 316 with open(self._md5_checksum_file, 'rb') as f: 317 file_md5_list = file_md5_pb2.FileChecksumList() 318 319 try: 320 file_md5_list.ParseFromString(f.read()) 321 except DecodeError: 322 logging.warning( 323 'Failed to parse the workspace md5 checksum file.') 324 return False 325 326 for file_md5 in file_md5_list.file_checksums: 327 abs_path = (Path(self._root_type_to_path[file_md5.root_type]) 328 .joinpath(file_md5.rel_path)) 329 if not abs_path.is_file(): 330 return False 331 if atest_utils.md5sum(abs_path) != file_md5.md5sum: 332 return False 333 return True 334 335 336class WorkspaceGenerator: 337 """Class for generating a Bazel workspace.""" 338 339 # pylint: disable=too-many-arguments 340 def __init__(self, 341 resource_manager: ResourceManager, 342 workspace_out_path: Path, 343 host_out_path: Path, 344 build_out_dir: Path, 345 mod_info: module_info.ModuleInfo, 346 jdk_path: Path=None, 347 enabled_features: Set[Features] = None, 348 ): 349 """Initializes the generator. 350 351 Args: 352 workspace_out_path: Path where the workspace will be output. 353 host_out_path: Path of the ANDROID_HOST_OUT. 354 build_out_dir: Path of OUT_DIR 355 mod_info: ModuleInfo object. 356 enabled_features: Set of enabled features. 357 """ 358 self.enabled_features = enabled_features or set() 359 self.resource_manager = resource_manager 360 self.workspace_out_path = workspace_out_path 361 self.host_out_path = host_out_path 362 self.build_out_dir = build_out_dir 363 self.mod_info = mod_info 364 self.path_to_package = {} 365 self.jdk_path = jdk_path 366 367 def generate(self): 368 """Generate a Bazel workspace. 369 370 If the workspace md5 checksum file doesn't exist or is stale, a new 371 workspace will be generated. Otherwise, the existing workspace will be 372 reused. 373 """ 374 start = time.time() 375 enabled_features_file = self.workspace_out_path.joinpath( 376 'atest_bazel_mode_enabled_features') 377 enabled_features_file_contents = '\n'.join(sorted( 378 f.name for f in self.enabled_features if f.affects_workspace)) 379 380 if self.workspace_out_path.exists(): 381 # Update the file with the set of the currently enabled features to 382 # make sure that changes are detected in the workspace checksum. 383 enabled_features_file.write_text(enabled_features_file_contents) 384 if self.resource_manager.check_affects_files_md5(): 385 return 386 387 # We raise an exception if rmtree fails to avoid leaving stale 388 # files in the workspace that could interfere with execution. 389 shutil.rmtree(self.workspace_out_path) 390 391 atest_utils.colorful_print("Generating Bazel workspace.\n", 392 constants.RED) 393 394 self._add_test_module_targets() 395 396 self.workspace_out_path.mkdir(parents=True) 397 self._generate_artifacts() 398 399 # Note that we write the set of enabled features despite having written 400 # it above since the workspace no longer exists at this point. 401 enabled_features_file.write_text(enabled_features_file_contents) 402 403 self.resource_manager.get_product_out_file_path( 404 self.mod_info.mod_info_file_path.relative_to( 405 self.resource_manager.get_product_out_file_path()), True) 406 self.resource_manager.register_file_with_abs_path( 407 enabled_features_file) 408 self.resource_manager.save_affects_files_md5() 409 metrics.LocalDetectEvent( 410 detect_type=DetectType.FULL_GENERATE_BAZEL_WORKSPACE_TIME, 411 result=int(time.time() - start)) 412 413 def _add_test_module_targets(self): 414 seen = set() 415 416 for name, info in self.mod_info.name_to_module_info.items(): 417 # Ignore modules that have a 'host_cross_' prefix since they are 418 # duplicates of existing modules. For example, 419 # 'host_cross_aapt2_tests' is a duplicate of 'aapt2_tests'. We also 420 # ignore modules with a '_32' suffix since these also are redundant 421 # given that modules have both 32 and 64-bit variants built by 422 # default. See b/77288544#comment6 and b/23566667 for more context. 423 if name.endswith("_32") or name.startswith("host_cross_"): 424 continue 425 426 if (Features.EXPERIMENTAL_DEVICE_DRIVEN_TEST in 427 self.enabled_features and 428 self.mod_info.is_device_driven_test(info)): 429 self._resolve_dependencies( 430 self._add_device_test_target(info, False), seen) 431 432 if self.mod_info.is_host_unit_test(info): 433 self._resolve_dependencies( 434 self._add_deviceless_test_target(info), seen) 435 elif (Features.EXPERIMENTAL_ROBOLECTRIC_TEST in 436 self.enabled_features and 437 self.mod_info.is_modern_robolectric_test(info)): 438 self._resolve_dependencies( 439 self._add_tradefed_robolectric_test_target(info), seen) 440 elif (Features.EXPERIMENTAL_HOST_DRIVEN_TEST in 441 self.enabled_features and 442 self.mod_info.is_host_driven_test(info)): 443 self._resolve_dependencies( 444 self._add_device_test_target(info, True), seen) 445 446 def _resolve_dependencies( 447 self, top_level_target: Target, seen: Set[Target]): 448 449 stack = [deque([top_level_target])] 450 451 while stack: 452 top = stack[-1] 453 454 if not top: 455 stack.pop() 456 continue 457 458 target = top.popleft() 459 460 # Note that we're relying on Python's default identity-based hash 461 # and equality methods. This is fine since we actually DO want 462 # reference-equality semantics for Target objects in this context. 463 if target in seen: 464 continue 465 466 seen.add(target) 467 468 next_top = deque() 469 470 for ref in target.dependencies(): 471 info = ref.info or self._get_module_info(ref.name) 472 ref.set(self._add_prebuilt_target(info)) 473 next_top.append(ref.target()) 474 475 stack.append(next_top) 476 477 def _add_device_test_target(self, info: Dict[str, Any], 478 is_host_driven: bool) -> Target: 479 package_name = self._get_module_path(info) 480 name_suffix = 'host' if is_host_driven else 'device' 481 name = f'{info[constants.MODULE_INFO_ID]}_{name_suffix}' 482 483 def create(): 484 return TestTarget.create_device_test_target( 485 name, 486 package_name, 487 info, 488 is_host_driven, 489 ) 490 491 return self._add_target(package_name, name, create) 492 493 def _add_deviceless_test_target(self, info: Dict[str, Any]) -> Target: 494 package_name = self._get_module_path(info) 495 name = f'{info[constants.MODULE_INFO_ID]}_host' 496 497 def create(): 498 return TestTarget.create_deviceless_test_target( 499 name, 500 package_name, 501 info, 502 ) 503 504 return self._add_target(package_name, name, create) 505 506 def _add_tradefed_robolectric_test_target( 507 self, info: Dict[str, Any]) -> Target: 508 package_name = self._get_module_path(info) 509 name = f'{info[constants.MODULE_INFO_ID]}_host' 510 511 return self._add_target( 512 package_name, 513 name, 514 lambda : TestTarget.create_tradefed_robolectric_test_target( 515 name, package_name, info, f'//{JDK_PACKAGE_NAME}:{JDK_NAME}') 516 ) 517 518 def _add_prebuilt_target(self, info: Dict[str, Any]) -> Target: 519 package_name = self._get_module_path(info) 520 name = info[constants.MODULE_INFO_ID] 521 522 def create(): 523 return SoongPrebuiltTarget.create( 524 self, 525 info, 526 package_name, 527 ) 528 529 return self._add_target(package_name, name, create) 530 531 def _add_target(self, package_path: str, target_name: str, 532 create_fn: Callable) -> Target: 533 534 package = self.path_to_package.get(package_path) 535 536 if not package: 537 package = Package(package_path) 538 self.path_to_package[package_path] = package 539 540 target = package.get_target(target_name) 541 542 if target: 543 return target 544 545 target = create_fn() 546 package.add_target(target) 547 548 return target 549 550 def _get_module_info(self, module_name: str) -> Dict[str, Any]: 551 info = self.mod_info.get_module_info(module_name) 552 553 if not info: 554 raise Exception(f'Could not find module `{module_name}` in' 555 f' module_info file') 556 557 return info 558 559 def _get_module_path(self, info: Dict[str, Any]) -> str: 560 mod_path = info.get(constants.MODULE_PATH) 561 562 if len(mod_path) < 1: 563 module_name = info['module_name'] 564 raise Exception(f'Module `{module_name}` does not have any path') 565 566 if len(mod_path) > 1: 567 module_name = info['module_name'] 568 # We usually have a single path but there are a few exceptions for 569 # modules like libLLVM_android and libclang_android. 570 # TODO(yangbill): Raise an exception for multiple paths once 571 # b/233581382 is resolved. 572 warnings.formatwarning = lambda msg, *args, **kwargs: f'{msg}\n' 573 warnings.warn( 574 f'Module `{module_name}` has more than one path: `{mod_path}`') 575 576 return mod_path[0] 577 578 def _generate_artifacts(self): 579 """Generate workspace files on disk.""" 580 581 self._create_base_files() 582 583 self._add_workspace_resource(src='rules', dst='bazel/rules') 584 self._add_workspace_resource(src='configs', dst='bazel/configs') 585 586 self._add_bazel_bootstrap_files() 587 588 # Symlink to package with toolchain definitions. 589 self._symlink(src='prebuilts/build-tools', 590 target='prebuilts/build-tools') 591 592 device_infra_path = 'vendor/google/tools/atest/device_infra' 593 if self.resource_manager.get_src_file_path(device_infra_path).exists(): 594 self._symlink(src=device_infra_path, 595 target=device_infra_path) 596 597 self._create_constants_file() 598 599 self._generate_robolectric_resources() 600 601 for package in self.path_to_package.values(): 602 package.generate(self.workspace_out_path) 603 604 def _generate_robolectric_resources(self): 605 if not self.jdk_path: 606 return 607 608 self._generate_jdk_resources() 609 self._generate_android_all_resources() 610 611 def _generate_jdk_resources(self): 612 # TODO(b/265596946): Create the JDK toolchain instead of using 613 # a filegroup. 614 return self._add_target( 615 JDK_PACKAGE_NAME, 616 JDK_NAME, 617 lambda : FilegroupTarget( 618 JDK_PACKAGE_NAME, JDK_NAME, 619 self.resource_manager.get_src_file_path(self.jdk_path)) 620 ) 621 622 def _generate_android_all_resources(self): 623 package_name = 'android-all' 624 name = 'android-all' 625 626 return self._add_target( 627 package_name, 628 name, 629 lambda : FilegroupTarget( 630 package_name, name, 631 self.host_out_path.joinpath(f'testcases/{name}')) 632 ) 633 634 def _symlink(self, *, src, target): 635 """Create a symbolic link in workspace pointing to source file/dir. 636 637 Args: 638 src: A string of a relative path to root of Android source tree. 639 This is the source file/dir path for which the symbolic link 640 will be created. 641 target: A string of a relative path to workspace root. This is the 642 target file/dir path where the symbolic link will be created. 643 """ 644 symlink = self.workspace_out_path.joinpath(target) 645 symlink.parent.mkdir(parents=True, exist_ok=True) 646 symlink.symlink_to(self.resource_manager.get_src_file_path(src)) 647 648 def _create_base_files(self): 649 self._add_workspace_resource(src='WORKSPACE', dst='WORKSPACE') 650 self._add_workspace_resource(src='bazelrc', dst='.bazelrc') 651 652 self.workspace_out_path.joinpath('BUILD.bazel').touch() 653 654 def _add_bazel_bootstrap_files(self): 655 self._symlink(src='tools/asuite/atest/bazel/resources/bazel.sh', 656 target='bazel.sh') 657 # TODO(b/256924541): Consolidate the JDK with the version the Roboleaf 658 # team uses. 659 self._symlink(src='prebuilts/jdk/jdk17/BUILD.bazel', 660 target='prebuilts/jdk/jdk17/BUILD.bazel') 661 self._symlink(src='prebuilts/jdk/jdk17/linux-x86', 662 target='prebuilts/jdk/jdk17/linux-x86') 663 self._symlink(src='prebuilts/bazel/linux-x86_64/bazel', 664 target='prebuilts/bazel/linux-x86_64/bazel') 665 666 def _add_workspace_resource(self, src, dst): 667 """Add resource to the given destination in workspace. 668 669 Args: 670 src: A string of a relative path to root of Bazel artifacts. This is 671 the source file/dir path that will be added to workspace. 672 dst: A string of a relative path to workspace root. This is the 673 destination file/dir path where the artifacts will be added. 674 """ 675 src = self.resource_manager.get_resource_file_path(src, True) 676 dst = self.workspace_out_path.joinpath(dst) 677 dst.parent.mkdir(parents=True, exist_ok=True) 678 679 if src.is_file(): 680 shutil.copy(src, dst) 681 else: 682 shutil.copytree(src, dst, 683 ignore=shutil.ignore_patterns('__init__.py')) 684 685 def _create_constants_file(self): 686 687 def variable_name(target_name): 688 return re.sub(r'[.-]', '_', target_name) + '_label' 689 690 targets = [] 691 seen = set() 692 693 for module_name in TestTarget.DEVICELESS_TEST_PREREQUISITES.union( 694 TestTarget.DEVICE_TEST_PREREQUISITES): 695 info = self.mod_info.get_module_info(module_name) 696 target = self._add_prebuilt_target(info) 697 self._resolve_dependencies(target, seen) 698 targets.append(target) 699 700 with self.workspace_out_path.joinpath( 701 'constants.bzl').open('w') as f: 702 writer = IndentWriter(f) 703 for target in targets: 704 writer.write_line( 705 '%s = "%s"' % 706 (variable_name(target.name()), target.qualified_name()) 707 ) 708 709 710def _get_resource_root(): 711 return Path(os.path.dirname(__file__)).joinpath('bazel/resources') 712 713 714class Package: 715 """Class for generating an entire Package on disk.""" 716 717 def __init__(self, path: str): 718 self.path = path 719 self.imports = defaultdict(set) 720 self.name_to_target = OrderedDict() 721 722 def add_target(self, target): 723 target_name = target.name() 724 725 if target_name in self.name_to_target: 726 raise Exception(f'Cannot add target `{target_name}` which already' 727 f' exists in package `{self.path}`') 728 729 self.name_to_target[target_name] = target 730 731 for i in target.required_imports(): 732 self.imports[i.bzl_package].add(i.symbol) 733 734 def generate(self, workspace_out_path: Path): 735 package_dir = workspace_out_path.joinpath(self.path) 736 package_dir.mkdir(parents=True, exist_ok=True) 737 738 self._create_filesystem_layout(package_dir) 739 self._write_build_file(package_dir) 740 741 def _create_filesystem_layout(self, package_dir: Path): 742 for target in self.name_to_target.values(): 743 target.create_filesystem_layout(package_dir) 744 745 def _write_build_file(self, package_dir: Path): 746 with package_dir.joinpath('BUILD.bazel').open('w') as f: 747 f.write('package(default_visibility = ["//visibility:public"])\n') 748 f.write('\n') 749 750 for bzl_package, symbols in sorted(self.imports.items()): 751 symbols_text = ', '.join('"%s"' % s for s in sorted(symbols)) 752 f.write(f'load("{bzl_package}", {symbols_text})\n') 753 754 for target in self.name_to_target.values(): 755 f.write('\n') 756 target.write_to_build_file(f) 757 758 def get_target(self, target_name: str) -> Target: 759 return self.name_to_target.get(target_name, None) 760 761 762@dataclasses.dataclass(frozen=True) 763class Import: 764 bzl_package: str 765 symbol: str 766 767 768@dataclasses.dataclass(frozen=True) 769class Config: 770 name: str 771 out_path: Path 772 773 774class ModuleRef: 775 776 @staticmethod 777 def for_info(info) -> ModuleRef: 778 return ModuleRef(info=info) 779 780 @staticmethod 781 def for_name(name) -> ModuleRef: 782 return ModuleRef(name=name) 783 784 def __init__(self, info=None, name=None): 785 self.info = info 786 self.name = name 787 self._target = None 788 789 def target(self) -> Target: 790 if not self._target: 791 target_name = self.info[constants.MODULE_INFO_ID] 792 raise Exception(f'Target not set for ref `{target_name}`') 793 794 return self._target 795 796 def set(self, target): 797 self._target = target 798 799 800class Target(ABC): 801 """Abstract class for a Bazel target.""" 802 803 @abstractmethod 804 def name(self) -> str: 805 pass 806 807 def package_name(self) -> str: 808 pass 809 810 def qualified_name(self) -> str: 811 return f'//{self.package_name()}:{self.name()}' 812 813 def required_imports(self) -> Set[Import]: 814 return set() 815 816 def supported_configs(self) -> Set[Config]: 817 return set() 818 819 def dependencies(self) -> List[ModuleRef]: 820 return [] 821 822 def write_to_build_file(self, f: IO): 823 pass 824 825 def create_filesystem_layout(self, package_dir: Path): 826 pass 827 828 829class FilegroupTarget(Target): 830 831 def __init__( 832 self, 833 package_name: str, 834 target_name: str, 835 srcs_root: Path 836 ): 837 self._package_name = package_name 838 self._target_name = target_name 839 self._srcs_root = srcs_root 840 841 def name(self) -> str: 842 return self._target_name 843 844 def package_name(self) -> str: 845 return self._package_name 846 847 def write_to_build_file(self, f: IO): 848 writer = IndentWriter(f) 849 build_file_writer = BuildFileWriter(writer) 850 851 writer.write_line('filegroup(') 852 853 with writer.indent(): 854 build_file_writer.write_string_attribute('name', self._target_name) 855 build_file_writer.write_glob_attribute( 856 'srcs', [f'{self._target_name}_files/**']) 857 858 writer.write_line(')') 859 860 def create_filesystem_layout(self, package_dir: Path): 861 symlink = package_dir.joinpath(f'{self._target_name}_files') 862 symlink.symlink_to(self._srcs_root) 863 864 865class TestTarget(Target): 866 """Class for generating a test target.""" 867 868 DEVICELESS_TEST_PREREQUISITES = frozenset({ 869 'adb', 870 'atest-tradefed', 871 'atest_script_help.sh', 872 'atest_tradefed.sh', 873 'tradefed', 874 'tradefed-test-framework', 875 'bazel-result-reporter' 876 }) 877 878 DEVICE_TEST_PREREQUISITES = frozenset(DEVICELESS_TEST_PREREQUISITES.union( 879 frozenset({ 880 'aapt', 881 'aapt2', 882 'compatibility-tradefed', 883 'vts-core-tradefed-harness', 884 }))) 885 886 @staticmethod 887 def create_deviceless_test_target(name: str, package_name: str, 888 info: Dict[str, Any]): 889 return TestTarget( 890 package_name, 891 'tradefed_deviceless_test', 892 { 893 'name': name, 894 'test': ModuleRef.for_info(info), 895 'module_name': info["module_name"], 896 'tags': info.get(constants.MODULE_TEST_OPTIONS_TAGS, []), 897 }, 898 TestTarget.DEVICELESS_TEST_PREREQUISITES, 899 ) 900 901 @staticmethod 902 def create_device_test_target(name: str, package_name: str, 903 info: Dict[str, Any], is_host_driven: bool): 904 rule = ('tradefed_host_driven_device_test' if is_host_driven 905 else 'tradefed_device_driven_test') 906 907 return TestTarget( 908 package_name, 909 rule, 910 { 911 'name': name, 912 'test': ModuleRef.for_info(info), 913 'module_name': info["module_name"], 914 'suites': set( 915 info.get(constants.MODULE_COMPATIBILITY_SUITES, [])), 916 'tradefed_deps': list(map( 917 ModuleRef.for_name, 918 info.get(constants.MODULE_HOST_DEPS, []))), 919 'tags': info.get(constants.MODULE_TEST_OPTIONS_TAGS, []), 920 }, 921 TestTarget.DEVICE_TEST_PREREQUISITES, 922 ) 923 924 @staticmethod 925 def create_tradefed_robolectric_test_target( 926 name: str, 927 package_name: str, 928 info: Dict[str, Any], 929 jdk_label: str 930 ): 931 return TestTarget( 932 package_name, 933 'tradefed_robolectric_test', 934 { 935 'name': name, 936 'test': ModuleRef.for_info(info), 937 'module_name': info["module_name"], 938 'tags': info.get(constants.MODULE_TEST_OPTIONS_TAGS, []), 939 'jdk' : jdk_label, 940 }, 941 TestTarget.DEVICELESS_TEST_PREREQUISITES, 942 ) 943 944 def __init__(self, package_name: str, rule_name: str, 945 attributes: Dict[str, Any], prerequisites=frozenset()): 946 self._attributes = attributes 947 self._package_name = package_name 948 self._rule_name = rule_name 949 self._prerequisites = prerequisites 950 951 def name(self) -> str: 952 return self._attributes['name'] 953 954 def package_name(self) -> str: 955 return self._package_name 956 957 def required_imports(self) -> Set[Import]: 958 return { Import('//bazel/rules:tradefed_test.bzl', self._rule_name) } 959 960 def dependencies(self) -> List[ModuleRef]: 961 prerequisite_refs = map(ModuleRef.for_name, self._prerequisites) 962 963 declared_dep_refs = [] 964 for value in self._attributes.values(): 965 if isinstance(value, Iterable): 966 declared_dep_refs.extend( 967 [dep for dep in value if isinstance(dep, ModuleRef)]) 968 elif isinstance(value, ModuleRef): 969 declared_dep_refs.append(value) 970 971 return declared_dep_refs + list(prerequisite_refs) 972 973 def write_to_build_file(self, f: IO): 974 prebuilt_target_name = self._attributes['test'].target( 975 ).qualified_name() 976 writer = IndentWriter(f) 977 build_file_writer = BuildFileWriter(writer) 978 979 writer.write_line(f'{self._rule_name}(') 980 981 with writer.indent(): 982 build_file_writer.write_string_attribute( 983 'name', self._attributes['name']) 984 985 build_file_writer.write_string_attribute( 986 'module_name', self._attributes['module_name']) 987 988 build_file_writer.write_string_attribute( 989 'test', prebuilt_target_name) 990 991 build_file_writer.write_label_list_attribute( 992 'tradefed_deps', self._attributes.get('tradefed_deps')) 993 994 build_file_writer.write_string_list_attribute( 995 'suites', sorted(self._attributes.get('suites', []))) 996 997 build_file_writer.write_string_list_attribute( 998 'tags', sorted(self._attributes.get('tags', []))) 999 1000 build_file_writer.write_label_attribute( 1001 'jdk', self._attributes.get('jdk', None)) 1002 1003 writer.write_line(')') 1004 1005 1006def _read_robolectric_jdk_path(test_xml_config_template: Path) -> Path: 1007 if not test_xml_config_template.is_file(): 1008 return None 1009 1010 xml_root = ET.parse(test_xml_config_template).getroot() 1011 option = xml_root.find(".//option[@name='java-folder']") 1012 jdk_path = Path(option.get('value', '')) 1013 1014 if not jdk_path.is_relative_to('prebuilts/jdk'): 1015 raise Exception(f'Failed to get "java-folder" from ' 1016 f'`{test_xml_config_template}`') 1017 1018 return jdk_path 1019 1020 1021class BuildFileWriter: 1022 """Class for writing BUILD files.""" 1023 1024 def __init__(self, underlying: IndentWriter): 1025 self._underlying = underlying 1026 1027 def write_string_attribute(self, attribute_name, value): 1028 if value is None: 1029 return 1030 1031 self._underlying.write_line(f'{attribute_name} = "{value}",') 1032 1033 def write_label_attribute(self, attribute_name: str, label_name: str): 1034 if label_name is None: 1035 return 1036 1037 self._underlying.write_line(f'{attribute_name} = "{label_name}",') 1038 1039 def write_string_list_attribute(self, attribute_name, values): 1040 if not values: 1041 return 1042 1043 self._underlying.write_line(f'{attribute_name} = [') 1044 1045 with self._underlying.indent(): 1046 for value in values: 1047 self._underlying.write_line(f'"{value}",') 1048 1049 self._underlying.write_line('],') 1050 1051 def write_label_list_attribute( 1052 self, attribute_name: str, modules: List[ModuleRef]): 1053 if not modules: 1054 return 1055 1056 self._underlying.write_line(f'{attribute_name} = [') 1057 1058 with self._underlying.indent(): 1059 for label in sorted(set( 1060 m.target().qualified_name() for m in modules)): 1061 self._underlying.write_line(f'"{label}",') 1062 1063 self._underlying.write_line('],') 1064 1065 def write_glob_attribute(self, attribute_name: str, patterns: List[str]): 1066 self._underlying.write_line(f'{attribute_name} = glob([') 1067 1068 with self._underlying.indent(): 1069 for pattern in patterns: 1070 self._underlying.write_line(f'"{pattern}",') 1071 1072 self._underlying.write_line(']),') 1073 1074 1075@dataclasses.dataclass(frozen=True) 1076class Dependencies: 1077 static_dep_refs: List[ModuleRef] 1078 runtime_dep_refs: List[ModuleRef] 1079 data_dep_refs: List[ModuleRef] 1080 device_data_dep_refs: List[ModuleRef] 1081 1082 1083class SoongPrebuiltTarget(Target): 1084 """Class for generating a Soong prebuilt target on disk.""" 1085 1086 @staticmethod 1087 def create(gen: WorkspaceGenerator, 1088 info: Dict[str, Any], 1089 package_name: str=''): 1090 module_name = info['module_name'] 1091 1092 configs = [ 1093 Config('host', gen.host_out_path), 1094 Config('device', gen.resource_manager.get_product_out_file_path()), 1095 ] 1096 1097 installed_paths = get_module_installed_paths( 1098 info, gen.resource_manager.get_src_file_path()) 1099 config_files = group_paths_by_config(configs, installed_paths) 1100 1101 # For test modules, we only create symbolic link to the 'testcases' 1102 # directory since the information in module-info is not accurate. 1103 if gen.mod_info.is_tradefed_testable_module(info): 1104 config_files = {c: [c.out_path.joinpath(f'testcases/{module_name}')] 1105 for c in config_files.keys()} 1106 1107 enabled_features = gen.enabled_features 1108 1109 return SoongPrebuiltTarget( 1110 info, 1111 package_name, 1112 config_files, 1113 Dependencies( 1114 static_dep_refs = find_static_dep_refs( 1115 gen.mod_info, info, configs, 1116 gen.resource_manager.get_src_file_path(), enabled_features), 1117 runtime_dep_refs = find_runtime_dep_refs( 1118 gen.mod_info, info, configs, 1119 gen.resource_manager.get_src_file_path(), enabled_features), 1120 data_dep_refs = find_data_dep_refs( 1121 gen.mod_info, info, configs, 1122 gen.resource_manager.get_src_file_path()), 1123 device_data_dep_refs = find_device_data_dep_refs(gen, info), 1124 ), 1125 [ 1126 c for c in configs if c.name in map( 1127 str.lower, info.get(constants.MODULE_SUPPORTED_VARIANTS, [])) 1128 ], 1129 ) 1130 1131 def __init__(self, 1132 info: Dict[str, Any], 1133 package_name: str, 1134 config_files: Dict[Config, List[Path]], 1135 deps: Dependencies, 1136 supported_configs: List[Config]): 1137 self._target_name = info[constants.MODULE_INFO_ID] 1138 self._module_name = info[constants.MODULE_NAME] 1139 self._package_name = package_name 1140 self.config_files = config_files 1141 self.deps = deps 1142 self.suites = info.get(constants.MODULE_COMPATIBILITY_SUITES, []) 1143 self._supported_configs = supported_configs 1144 1145 def name(self) -> str: 1146 return self._target_name 1147 1148 def package_name(self) -> str: 1149 return self._package_name 1150 1151 def required_imports(self) -> Set[Import]: 1152 return { 1153 Import('//bazel/rules:soong_prebuilt.bzl', self._rule_name()), 1154 } 1155 1156 @functools.lru_cache(maxsize=128) 1157 def supported_configs(self) -> Set[Config]: 1158 # We deduce the supported configs from the installed paths since the 1159 # build exports incorrect metadata for some module types such as 1160 # Robolectric. The information exported from the build is only used if 1161 # the module does not have any installed paths. 1162 # TODO(b/232929584): Remove this once all modules correctly export the 1163 # supported variants. 1164 supported_configs = set(self.config_files.keys()) 1165 if supported_configs: 1166 return supported_configs 1167 1168 return self._supported_configs 1169 1170 def dependencies(self) -> List[ModuleRef]: 1171 all_deps = set(self.deps.runtime_dep_refs) 1172 all_deps.update(self.deps.data_dep_refs) 1173 all_deps.update(self.deps.device_data_dep_refs) 1174 all_deps.update(self.deps.static_dep_refs) 1175 return list(all_deps) 1176 1177 def write_to_build_file(self, f: IO): 1178 writer = IndentWriter(f) 1179 build_file_writer = BuildFileWriter(writer) 1180 1181 writer.write_line(f'{self._rule_name()}(') 1182 1183 with writer.indent(): 1184 writer.write_line(f'name = "{self._target_name}",') 1185 writer.write_line(f'module_name = "{self._module_name}",') 1186 self._write_files_attribute(writer) 1187 self._write_deps_attribute(writer, 'static_deps', 1188 self.deps.static_dep_refs) 1189 self._write_deps_attribute(writer, 'runtime_deps', 1190 self.deps.runtime_dep_refs) 1191 self._write_deps_attribute(writer, 'data', self.deps.data_dep_refs) 1192 1193 build_file_writer.write_label_list_attribute( 1194 'device_data', self.deps.device_data_dep_refs) 1195 build_file_writer.write_string_list_attribute( 1196 'suites', sorted(self.suites)) 1197 1198 writer.write_line(')') 1199 1200 def create_filesystem_layout(self, package_dir: Path): 1201 prebuilts_dir = package_dir.joinpath(self._target_name) 1202 prebuilts_dir.mkdir() 1203 1204 for config, files in self.config_files.items(): 1205 config_prebuilts_dir = prebuilts_dir.joinpath(config.name) 1206 config_prebuilts_dir.mkdir() 1207 1208 for f in files: 1209 rel_path = f.relative_to(config.out_path) 1210 symlink = config_prebuilts_dir.joinpath(rel_path) 1211 symlink.parent.mkdir(parents=True, exist_ok=True) 1212 symlink.symlink_to(f) 1213 1214 def _rule_name(self): 1215 return ('soong_prebuilt' if self.config_files 1216 else 'soong_uninstalled_prebuilt') 1217 1218 def _write_files_attribute(self, writer: IndentWriter): 1219 if not self.config_files: 1220 return 1221 1222 writer.write('files = ') 1223 write_config_select( 1224 writer, 1225 self.config_files, 1226 lambda c, _: writer.write( 1227 f'glob(["{self._target_name}/{c.name}/**/*"])'), 1228 ) 1229 writer.write_line(',') 1230 1231 def _write_deps_attribute(self, writer, attribute_name, module_refs): 1232 config_deps = filter_configs( 1233 group_targets_by_config(r.target() for r in module_refs), 1234 self.supported_configs() 1235 ) 1236 1237 if not config_deps: 1238 return 1239 1240 for config in self.supported_configs(): 1241 config_deps.setdefault(config, []) 1242 1243 writer.write(f'{attribute_name} = ') 1244 write_config_select( 1245 writer, 1246 config_deps, 1247 lambda _, targets: write_target_list(writer, targets), 1248 ) 1249 writer.write_line(',') 1250 1251 1252def group_paths_by_config( 1253 configs: List[Config], paths: List[Path]) -> Dict[Config, List[Path]]: 1254 1255 config_files = defaultdict(list) 1256 1257 for f in paths: 1258 matching_configs = [ 1259 c for c in configs if _is_relative_to(f, c.out_path)] 1260 1261 if not matching_configs: 1262 continue 1263 1264 # The path can only appear in ANDROID_HOST_OUT for host target or 1265 # ANDROID_PRODUCT_OUT, but cannot appear in both. 1266 if len(matching_configs) > 1: 1267 raise Exception(f'Installed path `{f}` is not in' 1268 f' ANDROID_HOST_OUT or ANDROID_PRODUCT_OUT') 1269 1270 config_files[matching_configs[0]].append(f) 1271 1272 return config_files 1273 1274 1275def group_targets_by_config( 1276 targets: List[Target]) -> Dict[Config, List[Target]]: 1277 1278 config_to_targets = defaultdict(list) 1279 1280 for target in targets: 1281 for config in target.supported_configs(): 1282 config_to_targets[config].append(target) 1283 1284 return config_to_targets 1285 1286 1287def filter_configs( 1288 config_dict: Dict[Config, Any], configs: Set[Config],) -> Dict[Config, Any]: 1289 return { k: v for (k, v) in config_dict.items() if k in configs } 1290 1291 1292def _is_relative_to(path1: Path, path2: Path) -> bool: 1293 """Return True if the path is relative to another path or False.""" 1294 # Note that this implementation is required because Path.is_relative_to only 1295 # exists starting with Python 3.9. 1296 try: 1297 path1.relative_to(path2) 1298 return True 1299 except ValueError: 1300 return False 1301 1302 1303def get_module_installed_paths( 1304 info: Dict[str, Any], src_root_path: Path) -> List[Path]: 1305 1306 # Install paths in module-info are usually relative to the Android 1307 # source root ${ANDROID_BUILD_TOP}. When the output directory is 1308 # customized by the user however, the install paths are absolute. 1309 def resolve(install_path_string): 1310 install_path = Path(install_path_string) 1311 if not install_path.expanduser().is_absolute(): 1312 return src_root_path.joinpath(install_path) 1313 return install_path 1314 1315 return map(resolve, info.get(constants.MODULE_INSTALLED)) 1316 1317 1318def find_runtime_dep_refs( 1319 mod_info: module_info.ModuleInfo, 1320 info: module_info.Module, 1321 configs: List[Config], 1322 src_root_path: Path, 1323 enabled_features: List[Features], 1324) -> List[ModuleRef]: 1325 """Return module references for runtime dependencies.""" 1326 1327 # We don't use the `dependencies` module-info field for shared libraries 1328 # since it's ambiguous and could generate more targets and pull in more 1329 # dependencies than necessary. In particular, libraries that support both 1330 # static and dynamic linking could end up becoming runtime dependencies 1331 # even though the build specifies static linking. For example, if a target 1332 # 'T' is statically linked to 'U' which supports both variants, the latter 1333 # still appears as a dependency. Since we can't tell, this would result in 1334 # the shared library variant of 'U' being added on the library path. 1335 libs = set() 1336 libs.update(info.get(constants.MODULE_SHARED_LIBS, [])) 1337 libs.update(info.get(constants.MODULE_RUNTIME_DEPS, [])) 1338 1339 if Features.EXPERIMENTAL_JAVA_RUNTIME_DEPENDENCIES in enabled_features: 1340 libs.update(info.get(constants.MODULE_LIBS, [])) 1341 1342 runtime_dep_refs = _find_module_refs(mod_info, configs, src_root_path, libs) 1343 1344 runtime_library_class = {'RLIB_LIBRARIES', 'DYLIB_LIBRARIES'} 1345 # We collect rlibs even though they are technically static libraries since 1346 # they could refer to dylibs which are required at runtime. Generating 1347 # Bazel targets for these intermediate modules keeps the generator simple 1348 # and preserves the shape (isomorphic) of the Soong structure making the 1349 # workspace easier to debug. 1350 for dep_name in info.get(constants.MODULE_DEPENDENCIES, []): 1351 dep_info = mod_info.get_module_info(dep_name) 1352 if not dep_info: 1353 continue 1354 if not runtime_library_class.intersection( 1355 dep_info.get(constants.MODULE_CLASS, [])): 1356 continue 1357 runtime_dep_refs.append(ModuleRef.for_info(dep_info)) 1358 1359 return runtime_dep_refs 1360 1361 1362def find_data_dep_refs( 1363 mod_info: module_info.ModuleInfo, 1364 info: module_info.Module, 1365 configs: List[Config], 1366 src_root_path: Path, 1367) -> List[ModuleRef]: 1368 """Return module references for data dependencies.""" 1369 1370 return _find_module_refs(mod_info, 1371 configs, 1372 src_root_path, 1373 info.get(constants.MODULE_DATA_DEPS, [])) 1374 1375 1376def find_device_data_dep_refs( 1377 gen: WorkspaceGenerator, 1378 info: module_info.Module, 1379) -> List[ModuleRef]: 1380 """Return module references for device data dependencies.""" 1381 1382 return _find_module_refs( 1383 gen.mod_info, 1384 [Config('device', gen.resource_manager.get_product_out_file_path())], 1385 gen.resource_manager.get_src_file_path(), 1386 info.get(constants.MODULE_TARGET_DEPS, [])) 1387 1388 1389def find_static_dep_refs( 1390 mod_info: module_info.ModuleInfo, 1391 info: module_info.Module, 1392 configs: List[Config], 1393 src_root_path: Path, 1394 enabled_features: List[Features], 1395) -> List[ModuleRef]: 1396 """Return module references for static libraries.""" 1397 1398 if Features.EXPERIMENTAL_JAVA_RUNTIME_DEPENDENCIES not in enabled_features: 1399 return [] 1400 1401 static_libs = set() 1402 static_libs.update(info.get(constants.MODULE_STATIC_LIBS, [])) 1403 static_libs.update(info.get(constants.MODULE_STATIC_DEPS, [])) 1404 1405 return _find_module_refs(mod_info, 1406 configs, 1407 src_root_path, 1408 static_libs) 1409 1410 1411def _find_module_refs( 1412 mod_info: module_info.ModuleInfo, 1413 configs: List[Config], 1414 src_root_path: Path, 1415 module_names: List[str], 1416) -> List[ModuleRef]: 1417 """Return module references for modules.""" 1418 1419 module_refs = [] 1420 1421 for name in module_names: 1422 info = mod_info.get_module_info(name) 1423 if not info: 1424 continue 1425 1426 installed_paths = get_module_installed_paths(info, src_root_path) 1427 config_files = group_paths_by_config(configs, installed_paths) 1428 if not config_files: 1429 continue 1430 1431 module_refs.append(ModuleRef.for_info(info)) 1432 1433 return module_refs 1434 1435 1436class IndentWriter: 1437 1438 def __init__(self, f: IO): 1439 self._file = f 1440 self._indent_level = 0 1441 self._indent_string = 4 * ' ' 1442 self._indent_next = True 1443 1444 def write_line(self, text: str=''): 1445 if text: 1446 self.write(text) 1447 1448 self._file.write('\n') 1449 self._indent_next = True 1450 1451 def write(self, text): 1452 if self._indent_next: 1453 self._file.write(self._indent_string * self._indent_level) 1454 self._indent_next = False 1455 1456 self._file.write(text) 1457 1458 @contextlib.contextmanager 1459 def indent(self): 1460 self._indent_level += 1 1461 yield 1462 self._indent_level -= 1 1463 1464 1465def write_config_select( 1466 writer: IndentWriter, 1467 config_dict: Dict[Config, Any], 1468 write_value_fn: Callable, 1469): 1470 writer.write_line('select({') 1471 1472 with writer.indent(): 1473 for config, value in sorted( 1474 config_dict.items(), key=lambda c: c[0].name): 1475 1476 writer.write(f'"//bazel/rules:{config.name}": ') 1477 write_value_fn(config, value) 1478 writer.write_line(',') 1479 1480 writer.write('})') 1481 1482 1483def write_target_list(writer: IndentWriter, targets: List[Target]): 1484 writer.write_line('[') 1485 1486 with writer.indent(): 1487 for label in sorted(set(t.qualified_name() for t in targets)): 1488 writer.write_line(f'"{label}",') 1489 1490 writer.write(']') 1491 1492 1493def _decorate_find_method(mod_info, finder_method_func, host, enabled_features): 1494 """A finder_method decorator to override TestInfo properties.""" 1495 1496 def use_bazel_runner(finder_obj, test_id): 1497 test_infos = finder_method_func(finder_obj, test_id) 1498 if not test_infos: 1499 return test_infos 1500 for tinfo in test_infos: 1501 m_info = mod_info.get_module_info(tinfo.test_name) 1502 1503 # TODO(b/262200630): Refactor the duplicated logic in 1504 # _decorate_find_method() and _add_test_module_targets() to 1505 # determine whether a test should run with Atest Bazel Mode. 1506 1507 # Only enable modern Robolectric tests since those are the only ones 1508 # TF currently supports. 1509 if mod_info.is_modern_robolectric_test(m_info): 1510 if Features.EXPERIMENTAL_ROBOLECTRIC_TEST in enabled_features: 1511 tinfo.test_runner = BazelTestRunner.NAME 1512 continue 1513 1514 # Only run device-driven tests in Bazel mode when '--host' is not 1515 # specified and the feature is enabled. 1516 if not host and mod_info.is_device_driven_test(m_info): 1517 if Features.EXPERIMENTAL_DEVICE_DRIVEN_TEST in enabled_features: 1518 tinfo.test_runner = BazelTestRunner.NAME 1519 continue 1520 1521 if mod_info.is_suite_in_compatibility_suites( 1522 'host-unit-tests', m_info) or ( 1523 Features.EXPERIMENTAL_HOST_DRIVEN_TEST in enabled_features 1524 and mod_info.is_host_driven_test(m_info)): 1525 tinfo.test_runner = BazelTestRunner.NAME 1526 return test_infos 1527 return use_bazel_runner 1528 1529 1530def create_new_finder(mod_info: module_info.ModuleInfo, 1531 finder: test_finder_base.TestFinderBase, 1532 host: bool, 1533 enabled_features: List[Features]=None): 1534 """Create new test_finder_base.Finder with decorated find_method. 1535 1536 Args: 1537 mod_info: ModuleInfo object. 1538 finder: Test Finder class. 1539 host: Whether to run the host variant. 1540 enabled_features: List of enabled features. 1541 1542 Returns: 1543 List of ordered find methods. 1544 """ 1545 return test_finder_base.Finder(finder.test_finder_instance, 1546 _decorate_find_method( 1547 mod_info, 1548 finder.find_method, 1549 host, 1550 enabled_features or []), 1551 finder.finder_info) 1552 1553 1554class RunCommandError(subprocess.CalledProcessError): 1555 """CalledProcessError but including debug information when it fails.""" 1556 def __str__(self): 1557 return f'{super().__str__()}\n' \ 1558 f'stdout={self.stdout}\n\n' \ 1559 f'stderr={self.stderr}' 1560 1561 1562def default_run_command(args: List[str], cwd: Path) -> str: 1563 result = subprocess.run( 1564 args=args, 1565 cwd=cwd, 1566 text=True, 1567 capture_output=True, 1568 check=False, 1569 ) 1570 if result.returncode: 1571 # Provide a more detailed log message including stdout and stderr. 1572 raise RunCommandError(result.returncode, result.args, result.stdout, 1573 result.stderr) 1574 return result.stdout 1575 1576 1577@dataclasses.dataclass 1578class BuildMetadata: 1579 build_branch: str 1580 build_target: str 1581 1582 1583class BazelTestRunner(trb.TestRunnerBase): 1584 """Bazel Test Runner class.""" 1585 1586 NAME = 'BazelTestRunner' 1587 EXECUTABLE = 'none' 1588 1589 # pylint: disable=redefined-outer-name 1590 # pylint: disable=too-many-arguments 1591 def __init__(self, 1592 results_dir, 1593 mod_info: module_info.ModuleInfo, 1594 extra_args: Dict[str, Any]=None, 1595 src_top: Path=None, 1596 workspace_path: Path=None, 1597 run_command: Callable=default_run_command, 1598 build_metadata: BuildMetadata=None, 1599 env: Dict[str, str]=None, 1600 **kwargs): 1601 super().__init__(results_dir, **kwargs) 1602 self.mod_info = mod_info 1603 self.src_top = src_top or Path(os.environ.get( 1604 constants.ANDROID_BUILD_TOP)) 1605 self.starlark_file = _get_resource_root().joinpath( 1606 'format_as_soong_module_name.cquery') 1607 1608 self.bazel_workspace = workspace_path or get_bazel_workspace_dir() 1609 self.bazel_binary = self.bazel_workspace.joinpath( 1610 'bazel.sh') 1611 self.run_command = run_command 1612 self._extra_args = extra_args or {} 1613 self.build_metadata = build_metadata or get_default_build_metadata() 1614 self.env = env or os.environ 1615 1616 # pylint: disable=unused-argument 1617 def run_tests(self, test_infos, extra_args, reporter): 1618 """Run the list of test_infos. 1619 1620 Args: 1621 test_infos: List of TestInfo. 1622 extra_args: Dict of extra args to add to test run. 1623 reporter: An instance of result_report.ResultReporter. 1624 """ 1625 reporter.register_unsupported_runner(self.NAME) 1626 ret_code = ExitCode.SUCCESS 1627 1628 try: 1629 run_cmds = self.generate_run_commands(test_infos, extra_args) 1630 except AbortRunException as e: 1631 atest_utils.colorful_print(f'Stop running test(s): {e}', 1632 constants.RED) 1633 return ExitCode.ERROR 1634 1635 for run_cmd in run_cmds: 1636 subproc = self.run(run_cmd, output_to_stdout=True) 1637 ret_code |= self.wait_for_subprocess(subproc) 1638 return ret_code 1639 1640 def _get_feature_config_or_warn(self, feature, env_var_name): 1641 feature_config = self.env.get(env_var_name) 1642 if not feature_config: 1643 logging.warning( 1644 'Ignoring `%s` because the `%s`' 1645 ' environment variable is not set.', 1646 # pylint: disable=no-member 1647 feature, env_var_name 1648 ) 1649 return feature_config 1650 1651 def _get_bes_publish_args(self, feature): 1652 bes_publish_config = self._get_feature_config_or_warn( 1653 feature, 'ATEST_BAZEL_BES_PUBLISH_CONFIG') 1654 1655 if not bes_publish_config: 1656 return [] 1657 1658 branch = self.build_metadata.build_branch 1659 target = self.build_metadata.build_target 1660 1661 return [ 1662 f'--config={bes_publish_config}', 1663 f'--build_metadata=ab_branch={branch}', 1664 f'--build_metadata=ab_target={target}' 1665 ] 1666 1667 def _get_remote_args(self, feature): 1668 remote_config = self._get_feature_config_or_warn( 1669 feature, 'ATEST_BAZEL_REMOTE_CONFIG') 1670 if not remote_config: 1671 return [] 1672 return [f'--config={remote_config}'] 1673 1674 def host_env_check(self): 1675 """Check that host env has everything we need. 1676 1677 We actually can assume the host env is fine because we have the same 1678 requirements that atest has. Update this to check for android env vars 1679 if that changes. 1680 """ 1681 1682 def get_test_runner_build_reqs(self, test_infos) -> Set[str]: 1683 if not test_infos: 1684 return set() 1685 1686 deps_expression = ' + '.join( 1687 sorted(self.test_info_target_label(i) for i in test_infos) 1688 ) 1689 1690 with tempfile.NamedTemporaryFile() as query_file: 1691 with open(query_file.name, 'w', encoding='utf-8') as _query_file: 1692 _query_file.write(f'deps(tests({deps_expression}))') 1693 1694 query_args = [ 1695 str(self.bazel_binary), 1696 'cquery', 1697 f'--query_file={query_file.name}', 1698 '--output=starlark', 1699 f'--starlark:file={self.starlark_file}', 1700 ] 1701 1702 output = self.run_command(query_args, self.bazel_workspace) 1703 1704 targets = set() 1705 robolectric_tests = set(filter( 1706 self._is_robolectric_test_suite, 1707 [test.test_name for test in test_infos])) 1708 1709 modules_to_variant = _parse_cquery_output(output) 1710 1711 for module, variants in modules_to_variant.items(): 1712 1713 # Skip specifying the build variant for Robolectric test modules 1714 # since they are special. Soong builds them with the `target` 1715 # variant although are installed as 'host' modules. 1716 if module in robolectric_tests: 1717 targets.add(module) 1718 continue 1719 1720 targets.add(_soong_target_for_variants(module, variants)) 1721 1722 return targets 1723 1724 def _is_robolectric_test_suite(self, module_name: str) -> bool: 1725 return self.mod_info.is_robolectric_test_suite( 1726 self.mod_info.get_module_info(module_name)) 1727 1728 def test_info_target_label(self, test: test_info.TestInfo) -> str: 1729 module_name = test.test_name 1730 info = self.mod_info.get_module_info(module_name) 1731 package_name = info.get(constants.MODULE_PATH)[0] 1732 target_suffix = 'host' 1733 1734 if not self._extra_args.get( 1735 constants.HOST, 1736 False) and self.mod_info.is_device_driven_test(info): 1737 target_suffix = 'device' 1738 1739 return f'//{package_name}:{module_name}_{target_suffix}' 1740 1741 def _get_bazel_feature_args(self, feature, extra_args, generator): 1742 if feature not in extra_args.get('BAZEL_MODE_FEATURES', []): 1743 return [] 1744 return generator(feature) 1745 1746 # pylint: disable=unused-argument 1747 def generate_run_commands(self, test_infos, extra_args, port=None): 1748 """Generate a list of run commands from TestInfos. 1749 1750 Args: 1751 test_infos: A set of TestInfo instances. 1752 extra_args: A Dict of extra args to append. 1753 port: Optional. An int of the port number to send events to. 1754 1755 Returns: 1756 A list of run commands to run the tests. 1757 """ 1758 startup_options = '' 1759 bazelrc = self.env.get('ATEST_BAZELRC') 1760 1761 if bazelrc: 1762 startup_options = f'--bazelrc={bazelrc}' 1763 1764 target_patterns = ' '.join( 1765 self.test_info_target_label(i) for i in test_infos) 1766 1767 bazel_args = parse_args(test_infos, extra_args, self.mod_info) 1768 1769 bazel_args.extend( 1770 self._get_bazel_feature_args( 1771 Features.EXPERIMENTAL_BES_PUBLISH, 1772 extra_args, 1773 self._get_bes_publish_args)) 1774 bazel_args.extend( 1775 self._get_bazel_feature_args( 1776 Features.EXPERIMENTAL_REMOTE, 1777 extra_args, 1778 self._get_remote_args)) 1779 1780 # This is an alternative to shlex.join that doesn't exist in Python 1781 # versions < 3.8. 1782 bazel_args_str = ' '.join(shlex.quote(arg) for arg in bazel_args) 1783 1784 # Use 'cd' instead of setting the working directory in the subprocess 1785 # call for a working --dry-run command that users can run. 1786 return [ 1787 f'cd {self.bazel_workspace} &&' 1788 f'{self.bazel_binary} {startup_options} ' 1789 f'test {target_patterns} {bazel_args_str}' 1790 ] 1791 1792 1793def parse_args( 1794 test_infos: List[test_info.TestInfo], 1795 extra_args: Dict[str, Any], 1796 mod_info: module_info.ModuleInfo) -> Dict[str, Any]: 1797 """Parse commandline args and passes supported args to bazel. 1798 1799 Args: 1800 test_infos: A set of TestInfo instances. 1801 extra_args: A Dict of extra args to append. 1802 mod_info: A ModuleInfo object. 1803 1804 Returns: 1805 A list of args to append to the run command. 1806 """ 1807 1808 args_to_append = [] 1809 # Make a copy of the `extra_args` dict to avoid modifying it for other 1810 # Atest runners. 1811 extra_args_copy = extra_args.copy() 1812 1813 # Remove the `--host` flag since we already pass that in the rule's 1814 # implementation. 1815 extra_args_copy.pop(constants.HOST, None) 1816 1817 # Map args to their native Bazel counterparts. 1818 for arg in _SUPPORTED_BAZEL_ARGS: 1819 if arg not in extra_args_copy: 1820 continue 1821 args_to_append.extend( 1822 _map_to_bazel_args(arg, extra_args_copy[arg])) 1823 # Remove the argument since we already mapped it to a Bazel option 1824 # and no longer need it mapped to a Tradefed argument below. 1825 del extra_args_copy[arg] 1826 1827 # TODO(b/215461642): Store the extra_args in the top-level object so 1828 # that we don't have to re-parse the extra args to get BAZEL_ARG again. 1829 tf_args, _ = tfr.extra_args_to_tf_args( 1830 mod_info, test_infos, extra_args_copy) 1831 1832 # Add ATest include filter argument to allow testcase filtering. 1833 tf_args.extend(tfr.get_include_filter(test_infos)) 1834 1835 args_to_append.extend([f'--test_arg={i}' for i in tf_args]) 1836 1837 # Default to --test_output=errors unless specified otherwise 1838 if not any(arg.startswith('--test_output=') for arg in args_to_append): 1839 args_to_append.append('--test_output=errors') 1840 1841 return args_to_append 1842 1843def _map_to_bazel_args(arg: str, arg_value: Any) -> List[str]: 1844 return _SUPPORTED_BAZEL_ARGS[arg]( 1845 arg_value) if arg in _SUPPORTED_BAZEL_ARGS else [] 1846 1847 1848def _parse_cquery_output(output: str) -> Dict[str, Set[str]]: 1849 module_to_build_variants = defaultdict(set) 1850 1851 for line in filter(bool, map(str.strip, output.splitlines())): 1852 module_name, build_variant = line.split(':') 1853 module_to_build_variants[module_name].add(build_variant) 1854 1855 return module_to_build_variants 1856 1857 1858def _soong_target_for_variants( 1859 module_name: str, 1860 build_variants: Set[str]) -> str: 1861 1862 if not build_variants: 1863 raise Exception(f'Missing the build variants for module {module_name} ' 1864 f'in cquery output!') 1865 1866 if len(build_variants) > 1: 1867 return module_name 1868 1869 return f'{module_name}-{_CONFIG_TO_VARIANT[list(build_variants)[0]]}' 1870