1#!/usr/bin/env python 2# Copyright 2020 The gRPC Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# Script to extract build metadata from bazel BUILD. 17# To avoid having two sources of truth for the build metadata (build 18# targets, source files, header files etc.), this script analyzes the contents 19# of bazel BUILD files and generates a YAML file (currently called 20# build_autogenerated.yaml). The format and semantics of the generated YAML files 21# is chosen to match the format of a "build.yaml" file, which used 22# to be build the source of truth for gRPC build before bazel became 23# the primary build system. 24# A good basic overview of the "build.yaml" format is available here: 25# https://github.com/grpc/grpc/blob/master/templates/README.md. Note that 26# while useful as an overview, the doc does not act as formal spec 27# (formal spec does not exist in fact) and the doc can be incomplete, 28# inaccurate or slightly out of date. 29# TODO(jtattermusch): In the future we want to get rid of the legacy build.yaml 30# format entirely or simplify it to a point where it becomes self-explanatory 31# and doesn't need any detailed documentation. 32 33import subprocess 34import yaml 35import xml.etree.ElementTree as ET 36import os 37import sys 38import build_cleaner 39 40_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) 41os.chdir(_ROOT) 42 43 44def _bazel_query_xml_tree(query): 45 """Get xml output of bazel query invocation, parsed as XML tree""" 46 output = subprocess.check_output( 47 ['tools/bazel', 'query', '--noimplicit_deps', '--output', 'xml', query]) 48 return ET.fromstring(output) 49 50 51def _rule_dict_from_xml_node(rule_xml_node): 52 """Converts XML node representing a rule (obtained from "bazel query --output xml") to a dictionary that contains all the metadata we will need.""" 53 result = { 54 'class': rule_xml_node.attrib.get('class'), 55 'name': rule_xml_node.attrib.get('name'), 56 'srcs': [], 57 'hdrs': [], 58 'deps': [], 59 'data': [], 60 'tags': [], 61 'args': [], 62 'generator_function': None, 63 'size': None, 64 'flaky': False, 65 } 66 for child in rule_xml_node: 67 # all the metadata we want is stored under "list" tags 68 if child.tag == 'list': 69 list_name = child.attrib['name'] 70 if list_name in ['srcs', 'hdrs', 'deps', 'data', 'tags', 'args']: 71 result[list_name] += [item.attrib['value'] for item in child] 72 if child.tag == 'string': 73 string_name = child.attrib['name'] 74 if string_name in ['generator_function', 'size']: 75 result[string_name] = child.attrib['value'] 76 if child.tag == 'boolean': 77 bool_name = child.attrib['name'] 78 if bool_name in ['flaky']: 79 result[bool_name] = child.attrib['value'] == 'true' 80 return result 81 82 83def _extract_rules_from_bazel_xml(xml_tree): 84 """Extract bazel rules from an XML tree node obtained from "bazel query --output xml" command.""" 85 result = {} 86 for child in xml_tree: 87 if child.tag == 'rule': 88 rule_dict = _rule_dict_from_xml_node(child) 89 rule_clazz = rule_dict['class'] 90 rule_name = rule_dict['name'] 91 if rule_clazz in [ 92 'cc_library', 'cc_binary', 'cc_test', 'cc_proto_library', 93 'proto_library' 94 ]: 95 if rule_name in result: 96 raise Exception('Rule %s already present' % rule_name) 97 result[rule_name] = rule_dict 98 return result 99 100 101def _get_bazel_label(target_name): 102 if ':' in target_name: 103 return '//%s' % target_name 104 else: 105 return '//:%s' % target_name 106 107 108def _extract_source_file_path(label): 109 """Gets relative path to source file from bazel deps listing""" 110 if label.startswith('//'): 111 label = label[len('//'):] 112 # labels in form //:src/core/lib/surface/call_test_only.h 113 if label.startswith(':'): 114 label = label[len(':'):] 115 # labels in form //test/core/util:port.cc 116 label = label.replace(':', '/') 117 return label 118 119 120def _extract_public_headers(bazel_rule): 121 """Gets list of public headers from a bazel rule""" 122 result = [] 123 for dep in bazel_rule['hdrs']: 124 if dep.startswith('//:include/') and dep.endswith('.h'): 125 result.append(_extract_source_file_path(dep)) 126 return list(sorted(result)) 127 128 129def _extract_nonpublic_headers(bazel_rule): 130 """Gets list of non-public headers from a bazel rule""" 131 result = [] 132 for dep in bazel_rule['hdrs']: 133 if dep.startswith('//') and not dep.startswith( 134 '//:include/') and dep.endswith('.h'): 135 result.append(_extract_source_file_path(dep)) 136 return list(sorted(result)) 137 138 139def _extract_sources(bazel_rule): 140 """Gets list of source files from a bazel rule""" 141 result = [] 142 for dep in bazel_rule['srcs']: 143 if dep.startswith('//') and (dep.endswith('.cc') or dep.endswith('.c') 144 or dep.endswith('.proto')): 145 result.append(_extract_source_file_path(dep)) 146 return list(sorted(result)) 147 148 149def _extract_deps(bazel_rule): 150 """Gets list of deps from from a bazel rule""" 151 return list(sorted(bazel_rule['deps'])) 152 153 154def _create_target_from_bazel_rule(target_name, bazel_rules): 155 """Create build.yaml-like target definition from bazel metadata""" 156 bazel_rule = bazel_rules[_get_bazel_label(target_name)] 157 158 # Create a template for our target from the bazel rule. Initially we only 159 # populate some "private" fields with the original info we got from bazel 160 # and only later we will populate the public fields (once we do some extra 161 # postprocessing). 162 result = { 163 'name': target_name, 164 '_PUBLIC_HEADERS_BAZEL': _extract_public_headers(bazel_rule), 165 '_HEADERS_BAZEL': _extract_nonpublic_headers(bazel_rule), 166 '_SRC_BAZEL': _extract_sources(bazel_rule), 167 '_DEPS_BAZEL': _extract_deps(bazel_rule), 168 } 169 return result 170 171 172def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False): 173 """Sort library names to form correct build order. Use metadata from lib_dict""" 174 # we find correct build order by performing a topological sort 175 # expected output: if library B depends on A, A should be listed first 176 177 # all libs that are not in the dictionary are considered external. 178 external_deps = list( 179 sorted([lib_name for lib_name in lib_names if lib_name not in lib_dict 180 ])) 181 if verbose: 182 print('topo_ordering ' + str(lib_names)) 183 print(' external_deps ' + str(external_deps)) 184 185 result = list(external_deps) # external deps will be listed first 186 while len(result) < len(lib_names): 187 more_results = [] 188 for lib in lib_names: 189 if lib not in result: 190 dep_set = set(lib_dict[lib].get(deps_key_name, [])) 191 dep_set = dep_set.intersection(lib_names) 192 # if lib only depends on what's already built, add it to the results 193 if not dep_set.difference(set(result)): 194 more_results.append(lib) 195 if not more_results: 196 raise Exception( 197 'Cannot sort topologically, there seems to be a cyclic dependency' 198 ) 199 if verbose: 200 print(' adding ' + str(more_results)) 201 result = result + list( 202 sorted(more_results 203 )) # when build order doesn't matter, sort lexicographically 204 return result 205 206 207# TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a slightly different logic) 208def _populate_transitive_deps(bazel_rules): 209 """Add 'transitive_deps' field for each of the rules""" 210 transitive_deps = {} 211 for rule_name in bazel_rules.keys(): 212 transitive_deps[rule_name] = set(bazel_rules[rule_name]['deps']) 213 214 while True: 215 deps_added = 0 216 for rule_name in bazel_rules.keys(): 217 old_deps = transitive_deps[rule_name] 218 new_deps = set(old_deps) 219 for dep_name in old_deps: 220 new_deps.update(transitive_deps.get(dep_name, set())) 221 deps_added += len(new_deps) - len(old_deps) 222 transitive_deps[rule_name] = new_deps 223 # if none of the transitive dep sets has changed, we're done 224 if deps_added == 0: 225 break 226 227 for rule_name, bazel_rule in bazel_rules.items(): 228 bazel_rule['transitive_deps'] = list(sorted(transitive_deps[rule_name])) 229 230 231def _external_dep_name_from_bazel_dependency(bazel_dep): 232 """Returns name of dependency if external bazel dependency is provided or None""" 233 if bazel_dep.startswith('@com_google_absl//'): 234 # special case for add dependency on one of the absl libraries (there is not just one absl library) 235 prefixlen = len('@com_google_absl//') 236 return bazel_dep[prefixlen:] 237 elif bazel_dep == '//external:upb_lib': 238 return 'upb' 239 elif bazel_dep == '//external:benchmark': 240 return 'benchmark' 241 else: 242 # all the other external deps such as protobuf, cares, zlib 243 # don't need to be listed explicitly, they are handled automatically 244 # by the build system (make, cmake) 245 return None 246 247 248def _expand_intermediate_deps(target_dict, public_dep_names, bazel_rules): 249 # Some of the libraries defined by bazel won't be exposed in build.yaml 250 # We call these "intermediate" dependencies. This method expands 251 # the intermediate deps for given target (populates library's 252 # headers, sources and dicts as if the intermediate dependency never existed) 253 254 # use this dictionary to translate from bazel labels to dep names 255 bazel_label_to_dep_name = {} 256 for dep_name in public_dep_names: 257 bazel_label_to_dep_name[_get_bazel_label(dep_name)] = dep_name 258 259 target_name = target_dict['name'] 260 bazel_deps = target_dict['_DEPS_BAZEL'] 261 262 # initial values 263 public_headers = set(target_dict['_PUBLIC_HEADERS_BAZEL']) 264 headers = set(target_dict['_HEADERS_BAZEL']) 265 src = set(target_dict['_SRC_BAZEL']) 266 deps = set() 267 268 expansion_blacklist = set() 269 to_expand = set(bazel_deps) 270 while to_expand: 271 272 # start with the last dependency to be built 273 build_order = _sort_by_build_order(list(to_expand), bazel_rules, 274 'transitive_deps') 275 276 bazel_dep = build_order[-1] 277 to_expand.remove(bazel_dep) 278 279 is_public = bazel_dep in bazel_label_to_dep_name 280 external_dep_name_maybe = _external_dep_name_from_bazel_dependency( 281 bazel_dep) 282 283 if is_public: 284 # this is not an intermediate dependency we so we add it 285 # to the list of public dependencies to the list, in the right format 286 deps.add(bazel_label_to_dep_name[bazel_dep]) 287 288 # we do not want to expand any intermediate libraries that are already included 289 # by the dependency we just added 290 expansion_blacklist.update( 291 bazel_rules[bazel_dep]['transitive_deps']) 292 293 elif external_dep_name_maybe: 294 deps.add(external_dep_name_maybe) 295 296 elif bazel_dep.startswith( 297 '//external:') or not bazel_dep.startswith('//'): 298 # all the other external deps can be skipped 299 pass 300 301 elif bazel_dep in expansion_blacklist: 302 # do not expand if a public dependency that depends on this has already been expanded 303 pass 304 305 else: 306 if bazel_dep in bazel_rules: 307 # this is an intermediate library, expand it 308 public_headers.update( 309 _extract_public_headers(bazel_rules[bazel_dep])) 310 headers.update( 311 _extract_nonpublic_headers(bazel_rules[bazel_dep])) 312 src.update(_extract_sources(bazel_rules[bazel_dep])) 313 314 new_deps = _extract_deps(bazel_rules[bazel_dep]) 315 to_expand.update(new_deps) 316 else: 317 raise Exception(bazel_dep + ' not in bazel_rules') 318 319 # make the 'deps' field transitive, but only list non-intermediate deps and selected external deps 320 bazel_transitive_deps = bazel_rules[_get_bazel_label( 321 target_name)]['transitive_deps'] 322 for transitive_bazel_dep in bazel_transitive_deps: 323 public_name = bazel_label_to_dep_name.get(transitive_bazel_dep, None) 324 if public_name: 325 deps.add(public_name) 326 external_dep_name_maybe = _external_dep_name_from_bazel_dependency( 327 transitive_bazel_dep) 328 if external_dep_name_maybe: 329 # expanding all absl libraries is technically correct but creates too much noise 330 if not external_dep_name_maybe.startswith('absl'): 331 deps.add(external_dep_name_maybe) 332 333 target_dict['public_headers'] = list(sorted(public_headers)) 334 target_dict['headers'] = list(sorted(headers)) 335 target_dict['src'] = list(sorted(src)) 336 target_dict['deps'] = list(sorted(deps)) 337 338 339def _generate_build_metadata(build_extra_metadata, bazel_rules): 340 """Generate build metadata in build.yaml-like format bazel build metadata and build.yaml-specific "extra metadata".""" 341 lib_names = list(build_extra_metadata.keys()) 342 result = {} 343 344 for lib_name in lib_names: 345 lib_dict = _create_target_from_bazel_rule(lib_name, bazel_rules) 346 347 # Figure out the final list of headers and sources for given target. 348 # While this is mostly based on bazel build metadata, build.yaml does 349 # not necessarily expose all the targets that are present in bazel build. 350 # These "intermediate dependencies" might get flattened. 351 # TODO(jtattermusch): This is done to avoid introducing too many intermediate 352 # libraries into the build.yaml-based builds (which might in cause issues 353 # building language-specific artifacts) and also because the libraries 354 # in build.yaml-based build are generally considered units of distributions 355 # (= public libraries that are visible to the user and are installable), 356 # while in bazel builds it is customary to define larger number of smaller 357 # "sublibraries". The need for elision (and expansion) 358 # of intermediate libraries can be re-evaluated in the future. 359 _expand_intermediate_deps(lib_dict, lib_names, bazel_rules) 360 361 # populate extra properties from the build.yaml-specific "extra metadata" 362 lib_dict.update(build_extra_metadata.get(lib_name, {})) 363 364 # store to results 365 result[lib_name] = lib_dict 366 367 # Rename targets marked with "_RENAME" extra metadata. 368 # This is mostly a cosmetic change to ensure that we end up with build.yaml target 369 # names we're used to from the past (and also to avoid too long target names). 370 # The rename step needs to be made after we're done with most of processing logic 371 # otherwise the already-renamed libraries will have different names than expected 372 for lib_name in lib_names: 373 to_name = build_extra_metadata.get(lib_name, {}).get('_RENAME', None) 374 if to_name: 375 # store lib under the new name and also change its 'name' property 376 if to_name in result: 377 raise Exception('Cannot rename target ' + lib_name + ', ' + 378 to_name + ' already exists.') 379 lib_dict = result.pop(lib_name) 380 lib_dict['name'] = to_name 381 result[to_name] = lib_dict 382 383 # dep names need to be updated as well 384 for lib_dict_to_update in result.values(): 385 lib_dict_to_update['deps'] = list([ 386 to_name if dep == lib_name else dep 387 for dep in lib_dict_to_update['deps'] 388 ]) 389 390 # make sure deps are listed in reverse topological order (e.g. "grpc gpr" and not "gpr grpc") 391 for lib_dict in result.values(): 392 lib_dict['deps'] = list( 393 reversed(_sort_by_build_order(lib_dict['deps'], result, 'deps'))) 394 395 return result 396 397 398def _convert_to_build_yaml_like(lib_dict): 399 lib_names = [ 400 lib_name for lib_name in list(lib_dict.keys()) 401 if lib_dict[lib_name].get('_TYPE', 'library') == 'library' 402 ] 403 target_names = [ 404 lib_name for lib_name in list(lib_dict.keys()) 405 if lib_dict[lib_name].get('_TYPE', 'library') == 'target' 406 ] 407 test_names = [ 408 lib_name for lib_name in list(lib_dict.keys()) 409 if lib_dict[lib_name].get('_TYPE', 'library') == 'test' 410 ] 411 412 # list libraries and targets in predefined order 413 lib_list = [lib_dict[lib_name] for lib_name in lib_names] 414 target_list = [lib_dict[lib_name] for lib_name in target_names] 415 test_list = [lib_dict[lib_name] for lib_name in test_names] 416 417 # get rid of temporary private fields prefixed with "_" and some other useless fields 418 for lib in lib_list: 419 for field_to_remove in [k for k in lib.keys() if k.startswith('_')]: 420 lib.pop(field_to_remove, None) 421 for target in target_list: 422 for field_to_remove in [k for k in target.keys() if k.startswith('_')]: 423 target.pop(field_to_remove, None) 424 target.pop('public_headers', 425 None) # public headers make no sense for targets 426 for test in test_list: 427 for field_to_remove in [k for k in test.keys() if k.startswith('_')]: 428 test.pop(field_to_remove, None) 429 test.pop('public_headers', 430 None) # public headers make no sense for tests 431 432 build_yaml_like = { 433 'libs': lib_list, 434 'filegroups': [], 435 'targets': target_list, 436 'tests': test_list, 437 } 438 return build_yaml_like 439 440 441def _extract_cc_tests(bazel_rules): 442 """Gets list of cc_test tests from bazel rules""" 443 result = [] 444 for bazel_rule in bazel_rules.values(): 445 if bazel_rule['class'] == 'cc_test': 446 test_name = bazel_rule['name'] 447 if test_name.startswith('//'): 448 prefixlen = len('//') 449 result.append(test_name[prefixlen:]) 450 return list(sorted(result)) 451 452 453def _exclude_unwanted_cc_tests(tests): 454 """Filters out bazel tests that we don't want to run with other build systems or we cannot build them reasonably""" 455 456 # most qps tests are autogenerated, we are fine without them 457 tests = [test for test in tests if not test.startswith('test/cpp/qps:')] 458 459 # we have trouble with census dependency outside of bazel 460 tests = [ 461 test for test in tests 462 if not test.startswith('test/cpp/ext/filters/census:') 463 ] 464 tests = [ 465 test for test in tests 466 if not test.startswith('test/cpp/microbenchmarks:bm_opencensus_plugin') 467 ] 468 469 # missing opencensus/stats/stats.h 470 tests = [ 471 test for test in tests if not test.startswith( 472 'test/cpp/end2end:server_load_reporting_end2end_test') 473 ] 474 tests = [ 475 test for test in tests if not test.startswith( 476 'test/cpp/server/load_reporter:lb_load_reporter_test') 477 ] 478 479 # The test uses --running_under_bazel cmdline argument 480 # To avoid the trouble needing to adjust it, we just skip the test 481 tests = [ 482 test for test in tests if not test.startswith( 483 'test/cpp/naming:resolver_component_tests_runner_invoker') 484 ] 485 486 # the test requires 'client_crash_test_server' to be built 487 tests = [ 488 test for test in tests 489 if not test.startswith('test/cpp/end2end:time_change_test') 490 ] 491 492 # the test requires 'client_crash_test_server' to be built 493 tests = [ 494 test for test in tests 495 if not test.startswith('test/cpp/end2end:client_crash_test') 496 ] 497 498 # the test requires 'server_crash_test_client' to be built 499 tests = [ 500 test for test in tests 501 if not test.startswith('test/cpp/end2end:server_crash_test') 502 ] 503 504 # test never existed under build.yaml and it fails -> skip it 505 tests = [ 506 test for test in tests 507 if not test.startswith('test/core/tsi:ssl_session_cache_test') 508 ] 509 510 # the binary of this test does not get built with cmake 511 tests = [ 512 test for test in tests 513 if not test.startswith('test/cpp/util:channelz_sampler_test') 514 ] 515 516 return tests 517 518 519def _generate_build_extra_metadata_for_tests(tests, bazel_rules): 520 """For given tests, generate the "extra metadata" that we need for our "build.yaml"-like output. The extra metadata is generated from the bazel rule metadata by using a bunch of heuristics.""" 521 test_metadata = {} 522 for test in tests: 523 test_dict = {'build': 'test', '_TYPE': 'target'} 524 525 bazel_rule = bazel_rules[_get_bazel_label(test)] 526 527 bazel_tags = bazel_rule['tags'] 528 if 'manual' in bazel_tags: 529 # don't run the tests marked as "manual" 530 test_dict['run'] = False 531 532 if bazel_rule['flaky']: 533 # don't run tests that are marked as "flaky" under bazel 534 # because that would only add noise for the run_tests.py tests 535 # and seeing more failures for tests that we already know are flaky 536 # doesn't really help anything 537 test_dict['run'] = False 538 539 if 'no_uses_polling' in bazel_tags: 540 test_dict['uses_polling'] = False 541 542 if 'grpc_fuzzer' == bazel_rule['generator_function']: 543 # currently we hand-list fuzzers instead of generating them automatically 544 # because there's no way to obtain maxlen property from bazel BUILD file. 545 print('skipping fuzzer ' + test) 546 continue 547 548 # if any tags that restrict platform compatibility are present, 549 # generate the "platforms" field accordingly 550 # TODO(jtattermusch): there is also a "no_linux" tag, but we cannot take 551 # it into account as it is applied by grpc_cc_test when poller expansion 552 # is made (for tests where uses_polling=True). So for now, we just 553 # assume all tests are compatible with linux and ignore the "no_linux" tag 554 # completely. 555 known_platform_tags = set(['no_windows', 'no_mac']) 556 if set(bazel_tags).intersection(known_platform_tags): 557 platforms = [] 558 # assume all tests are compatible with linux and posix 559 platforms.append('linux') 560 platforms.append( 561 'posix') # there is no posix-specific tag in bazel BUILD 562 if not 'no_mac' in bazel_tags: 563 platforms.append('mac') 564 if not 'no_windows' in bazel_tags: 565 platforms.append('windows') 566 test_dict['platforms'] = platforms 567 568 if '//external:benchmark' in bazel_rule['transitive_deps']: 569 test_dict['benchmark'] = True 570 test_dict['defaults'] = 'benchmark' 571 572 cmdline_args = bazel_rule['args'] 573 if cmdline_args: 574 test_dict['args'] = list(cmdline_args) 575 576 uses_gtest = '//external:gtest' in bazel_rule['transitive_deps'] 577 if uses_gtest: 578 test_dict['gtest'] = True 579 580 if test.startswith('test/cpp') or uses_gtest: 581 test_dict['language'] = 'c++' 582 583 elif test.startswith('test/core'): 584 test_dict['language'] = 'c' 585 else: 586 raise Exception('wrong test' + test) 587 588 # short test name without the path. 589 # There can be name collisions, but we will resolve them later 590 simple_test_name = os.path.basename(_extract_source_file_path(test)) 591 test_dict['_RENAME'] = simple_test_name 592 593 test_metadata[test] = test_dict 594 595 # detect duplicate test names 596 tests_by_simple_name = {} 597 for test_name, test_dict in test_metadata.items(): 598 simple_test_name = test_dict['_RENAME'] 599 if not simple_test_name in tests_by_simple_name: 600 tests_by_simple_name[simple_test_name] = [] 601 tests_by_simple_name[simple_test_name].append(test_name) 602 603 # choose alternative names for tests with a name collision 604 for collision_list in tests_by_simple_name.values(): 605 if len(collision_list) > 1: 606 for test_name in collision_list: 607 long_name = test_name.replace('/', '_').replace(':', '_') 608 print( 609 'short name of "%s" collides with another test, renaming to %s' 610 % (test_name, long_name)) 611 test_metadata[test_name]['_RENAME'] = long_name 612 613 return test_metadata 614 615 616def _detect_and_print_issues(build_yaml_like): 617 """Try detecting some unusual situations and warn about them.""" 618 for tgt in build_yaml_like['targets']: 619 if tgt['build'] == 'test': 620 for src in tgt['src']: 621 if src.startswith('src/') and not src.endswith('.proto'): 622 print('source file from under "src/" tree used in test ' + 623 tgt['name'] + ': ' + src) 624 625 626# extra metadata that will be used to construct build.yaml 627# there are mostly extra properties that we weren't able to obtain from the bazel build 628# _TYPE: whether this is library, target or test 629# _RENAME: whether this target should be renamed to a different name (to match expectations of make and cmake builds) 630# NOTE: secure is 'check' by default, so setting secure = False below does matter 631_BUILD_EXTRA_METADATA = { 632 'third_party/address_sorting:address_sorting': { 633 'language': 'c', 634 'build': 'all', 635 'secure': False, 636 '_RENAME': 'address_sorting' 637 }, 638 'gpr': { 639 'language': 'c', 640 'build': 'all', 641 'secure': False 642 }, 643 'grpc': { 644 'language': 'c', 645 'build': 'all', 646 'baselib': True, 647 'secure': True, 648 'deps_linkage': 'static', 649 'dll': True, 650 'generate_plugin_registry': True 651 }, 652 'grpc++': { 653 'language': 'c++', 654 'build': 'all', 655 'baselib': True, 656 'dll': True 657 }, 658 'grpc++_alts': { 659 'language': 'c++', 660 'build': 'all', 661 'baselib': True 662 }, 663 'grpc++_error_details': { 664 'language': 'c++', 665 'build': 'all' 666 }, 667 'grpc++_reflection': { 668 'language': 'c++', 669 'build': 'all' 670 }, 671 'grpc++_unsecure': { 672 'language': 'c++', 673 'build': 'all', 674 'baselib': True, 675 'secure': False, 676 'dll': True 677 }, 678 # TODO(jtattermusch): do we need to set grpc_csharp_ext's LDFLAGS for wrapping memcpy in the same way as in build.yaml? 679 'grpc_csharp_ext': { 680 'language': 'c', 681 'build': 'all', 682 'deps_linkage': 'static', 683 'dll': 'only' 684 }, 685 'grpc_unsecure': { 686 'language': 'c', 687 'build': 'all', 688 'baselib': True, 689 'secure': False, 690 'deps_linkage': 'static', 691 'dll': True, 692 'generate_plugin_registry': True 693 }, 694 'grpcpp_channelz': { 695 'language': 'c++', 696 'build': 'all' 697 }, 698 'grpc++_test': { 699 'language': 'c++', 700 'build': 'private', 701 }, 702 'src/compiler:grpc_plugin_support': { 703 'language': 'c++', 704 'build': 'protoc', 705 'secure': False, 706 '_RENAME': 'grpc_plugin_support' 707 }, 708 'src/compiler:grpc_cpp_plugin': { 709 'language': 'c++', 710 'build': 'protoc', 711 'secure': False, 712 '_TYPE': 'target', 713 '_RENAME': 'grpc_cpp_plugin' 714 }, 715 'src/compiler:grpc_csharp_plugin': { 716 'language': 'c++', 717 'build': 'protoc', 718 'secure': False, 719 '_TYPE': 'target', 720 '_RENAME': 'grpc_csharp_plugin' 721 }, 722 'src/compiler:grpc_node_plugin': { 723 'language': 'c++', 724 'build': 'protoc', 725 'secure': False, 726 '_TYPE': 'target', 727 '_RENAME': 'grpc_node_plugin' 728 }, 729 'src/compiler:grpc_objective_c_plugin': { 730 'language': 'c++', 731 'build': 'protoc', 732 'secure': False, 733 '_TYPE': 'target', 734 '_RENAME': 'grpc_objective_c_plugin' 735 }, 736 'src/compiler:grpc_php_plugin': { 737 'language': 'c++', 738 'build': 'protoc', 739 'secure': False, 740 '_TYPE': 'target', 741 '_RENAME': 'grpc_php_plugin' 742 }, 743 'src/compiler:grpc_python_plugin': { 744 'language': 'c++', 745 'build': 'protoc', 746 'secure': False, 747 '_TYPE': 'target', 748 '_RENAME': 'grpc_python_plugin' 749 }, 750 'src/compiler:grpc_ruby_plugin': { 751 'language': 'c++', 752 'build': 'protoc', 753 'secure': False, 754 '_TYPE': 'target', 755 '_RENAME': 'grpc_ruby_plugin' 756 }, 757 758 # TODO(jtattermusch): consider adding grpc++_core_stats 759 760 # test support libraries 761 'test/core/util:grpc_test_util': { 762 'language': 'c', 763 'build': 'private', 764 '_RENAME': 'grpc_test_util' 765 }, 766 'test/core/util:grpc_test_util_unsecure': { 767 'language': 'c', 768 'build': 'private', 769 'secure': False, 770 '_RENAME': 'grpc_test_util_unsecure' 771 }, 772 # TODO(jtattermusch): consider adding grpc++_test_util_unsecure - it doesn't seem to be used by bazel build (don't forget to set secure: False) 773 'test/cpp/util:test_config': { 774 'language': 'c++', 775 'build': 'private', 776 '_RENAME': 'grpc++_test_config' 777 }, 778 'test/cpp/util:test_util': { 779 'language': 'c++', 780 'build': 'private', 781 '_RENAME': 'grpc++_test_util' 782 }, 783 784 # end2end test support libraries 785 'test/core/end2end:end2end_tests': { 786 'language': 'c', 787 'build': 'private', 788 'secure': True, 789 '_RENAME': 'end2end_tests' 790 }, 791 'test/core/end2end:end2end_nosec_tests': { 792 'language': 'c', 793 'build': 'private', 794 'secure': False, 795 '_RENAME': 'end2end_nosec_tests' 796 }, 797 798 # benchmark support libraries 799 'test/cpp/microbenchmarks:helpers': { 800 'language': 'c++', 801 'build': 'test', 802 'defaults': 'benchmark', 803 '_RENAME': 'benchmark_helpers' 804 }, 805 'test/cpp/interop:interop_client': { 806 'language': 'c++', 807 'build': 'test', 808 'run': False, 809 '_TYPE': 'target', 810 '_RENAME': 'interop_client' 811 }, 812 'test/cpp/interop:interop_server': { 813 'language': 'c++', 814 'build': 'test', 815 'run': False, 816 '_TYPE': 'target', 817 '_RENAME': 'interop_server' 818 }, 819 'test/cpp/interop:xds_interop_client': { 820 'language': 'c++', 821 'build': 'test', 822 'run': False, 823 '_TYPE': 'target', 824 '_RENAME': 'xds_interop_client' 825 }, 826 'test/cpp/interop:xds_interop_server': { 827 'language': 'c++', 828 'build': 'test', 829 'run': False, 830 '_TYPE': 'target', 831 '_RENAME': 'xds_interop_server' 832 }, 833 'test/cpp/interop:http2_client': { 834 'language': 'c++', 835 'build': 'test', 836 'run': False, 837 '_TYPE': 'target', 838 '_RENAME': 'http2_client' 839 }, 840 'test/cpp/qps:qps_json_driver': { 841 'language': 'c++', 842 'build': 'test', 843 'run': False, 844 '_TYPE': 'target', 845 '_RENAME': 'qps_json_driver' 846 }, 847 'test/cpp/qps:qps_worker': { 848 'language': 'c++', 849 'build': 'test', 850 'run': False, 851 '_TYPE': 'target', 852 '_RENAME': 'qps_worker' 853 }, 854 'test/cpp/util:grpc_cli': { 855 'language': 'c++', 856 'build': 'test', 857 'run': False, 858 '_TYPE': 'target', 859 '_RENAME': 'grpc_cli' 860 }, 861 862 # TODO(jtattermusch): create_jwt and verify_jwt breaks distribtests because it depends on grpc_test_utils and thus requires tests to be built 863 # For now it's ok to disable them as these binaries aren't very useful anyway. 864 #'test/core/security:create_jwt': { 'language': 'c', 'build': 'tool', '_TYPE': 'target', '_RENAME': 'grpc_create_jwt' }, 865 #'test/core/security:verify_jwt': { 'language': 'c', 'build': 'tool', '_TYPE': 'target', '_RENAME': 'grpc_verify_jwt' }, 866 867 # TODO(jtattermusch): add remaining tools such as grpc_print_google_default_creds_token (they are not used by bazel build) 868 869 # Fuzzers 870 'test/core/security:alts_credentials_fuzzer': { 871 'language': 'c++', 872 'build': 'fuzzer', 873 'corpus_dirs': ['test/core/security/corpus/alts_credentials_corpus'], 874 'maxlen': 2048, 875 '_TYPE': 'target', 876 '_RENAME': 'alts_credentials_fuzzer' 877 }, 878 'test/core/end2end/fuzzers:client_fuzzer': { 879 'language': 'c++', 880 'build': 'fuzzer', 881 'corpus_dirs': ['test/core/end2end/fuzzers/client_fuzzer_corpus'], 882 'maxlen': 2048, 883 'dict': 'test/core/end2end/fuzzers/hpack.dictionary', 884 '_TYPE': 'target', 885 '_RENAME': 'client_fuzzer' 886 }, 887 'test/core/transport/chttp2:hpack_parser_fuzzer': { 888 'language': 'c++', 889 'build': 'fuzzer', 890 'corpus_dirs': ['test/core/transport/chttp2/hpack_parser_corpus'], 891 'maxlen': 512, 892 'dict': 'test/core/end2end/fuzzers/hpack.dictionary', 893 '_TYPE': 'target', 894 '_RENAME': 'hpack_parser_fuzzer_test' 895 }, 896 'test/core/http:request_fuzzer': { 897 'language': 'c++', 898 'build': 'fuzzer', 899 'corpus_dirs': ['test/core/http/request_corpus'], 900 'maxlen': 2048, 901 '_TYPE': 'target', 902 '_RENAME': 'http_request_fuzzer_test' 903 }, 904 'test/core/http:response_fuzzer': { 905 'language': 'c++', 906 'build': 'fuzzer', 907 'corpus_dirs': ['test/core/http/response_corpus'], 908 'maxlen': 2048, 909 '_TYPE': 'target', 910 '_RENAME': 'http_response_fuzzer_test' 911 }, 912 'test/core/json:json_fuzzer': { 913 'language': 'c++', 914 'build': 'fuzzer', 915 'corpus_dirs': ['test/core/json/corpus'], 916 'maxlen': 512, 917 '_TYPE': 'target', 918 '_RENAME': 'json_fuzzer_test' 919 }, 920 'test/core/nanopb:fuzzer_response': { 921 'language': 'c++', 922 'build': 'fuzzer', 923 'corpus_dirs': ['test/core/nanopb/corpus_response'], 924 'maxlen': 128, 925 '_TYPE': 'target', 926 '_RENAME': 'nanopb_fuzzer_response_test' 927 }, 928 'test/core/nanopb:fuzzer_serverlist': { 929 'language': 'c++', 930 'build': 'fuzzer', 931 'corpus_dirs': ['test/core/nanopb/corpus_serverlist'], 932 'maxlen': 128, 933 '_TYPE': 'target', 934 '_RENAME': 'nanopb_fuzzer_serverlist_test' 935 }, 936 'test/core/slice:percent_decode_fuzzer': { 937 'language': 'c++', 938 'build': 'fuzzer', 939 'corpus_dirs': ['test/core/slice/percent_decode_corpus'], 940 'maxlen': 32, 941 '_TYPE': 'target', 942 '_RENAME': 'percent_decode_fuzzer' 943 }, 944 'test/core/slice:percent_encode_fuzzer': { 945 'language': 'c++', 946 'build': 'fuzzer', 947 'corpus_dirs': ['test/core/slice/percent_encode_corpus'], 948 'maxlen': 32, 949 '_TYPE': 'target', 950 '_RENAME': 'percent_encode_fuzzer' 951 }, 952 'test/core/end2end/fuzzers:server_fuzzer': { 953 'language': 'c++', 954 'build': 'fuzzer', 955 'corpus_dirs': ['test/core/end2end/fuzzers/server_fuzzer_corpus'], 956 'maxlen': 2048, 957 'dict': 'test/core/end2end/fuzzers/hpack.dictionary', 958 '_TYPE': 'target', 959 '_RENAME': 'server_fuzzer' 960 }, 961 'test/core/security:ssl_server_fuzzer': { 962 'language': 'c++', 963 'build': 'fuzzer', 964 'corpus_dirs': ['test/core/security/corpus/ssl_server_corpus'], 965 'maxlen': 2048, 966 '_TYPE': 'target', 967 '_RENAME': 'ssl_server_fuzzer' 968 }, 969 'test/core/uri:uri_fuzzer_test': { 970 'language': 'c++', 971 'build': 'fuzzer', 972 'corpus_dirs': ['test/core/uri/uri_corpus'], 973 'maxlen': 128, 974 '_TYPE': 'target', 975 '_RENAME': 'uri_fuzzer_test' 976 }, 977 978 # TODO(jtattermusch): these fuzzers had no build.yaml equivalent 979 # test/core/compression:message_compress_fuzzer 980 # test/core/compression:message_decompress_fuzzer 981 # test/core/compression:stream_compression_fuzzer 982 # test/core/compression:stream_decompression_fuzzer 983 # test/core/slice:b64_decode_fuzzer 984 # test/core/slice:b64_encode_fuzzer 985} 986 987# We need a complete picture of all the targets and dependencies we're interested in 988# so we run multiple bazel queries and merge the results. 989_BAZEL_DEPS_QUERIES = [ 990 'deps("//test/...")', 991 'deps("//:all")', 992 'deps("//src/compiler/...")', 993 'deps("//src/proto/...")', 994] 995 996# Step 1: run a bunch of "bazel query --output xml" queries to collect 997# the raw build metadata from the bazel build. 998# At the end of this step we will have a dictionary of bazel rules 999# that are interesting to us (libraries, binaries, etc.) along 1000# with their most important metadata (sources, headers, dependencies) 1001# 1002# Example of a single bazel rule after being populated: 1003# '//:grpc' : { 'class': 'cc_library', 1004# 'hdrs': ['//:include/grpc/byte_buffer.h', ... ], 1005# 'srcs': ['//:src/core/lib/surface/init.cc', ... ], 1006# 'deps': ['//:grpc_common', ...], 1007# ... } 1008bazel_rules = {} 1009for query in _BAZEL_DEPS_QUERIES: 1010 bazel_rules.update( 1011 _extract_rules_from_bazel_xml(_bazel_query_xml_tree(query))) 1012 1013# Step 1a: Knowing the transitive closure of dependencies will make 1014# the postprocessing simpler, so compute the info for all our rules. 1015# 1016# Example: 1017# '//:grpc' : { ..., 1018# 'transitive_deps': ['//:gpr_base', ...] } 1019_populate_transitive_deps(bazel_rules) 1020 1021# Step 2: Extract the known bazel cc_test tests. While most tests 1022# will be buildable with other build systems just fine, some of these tests 1023# would be too difficult to build and run with other build systems, 1024# so we simply exclude the ones we don't want. 1025# Note that while making tests buildable with other build systems 1026# than just bazel is extra effort, we still need to do that for these 1027# reasons: 1028# - If our cmake build doesn't have any tests at all, it's hard to make 1029# sure that what it built actually works (we need at least some "smoke tests"). 1030# This is quite important because the build flags between bazel / non-bazel flag might differ 1031# (sometimes it's for interesting reasons that are not easy to overcome) 1032# which makes it even more important to have at least some tests for cmake/make 1033# - Our portability suite actually runs cmake tests and migration of portability 1034# suite fully towards bazel might be intricate (e.g. it's unclear whether it's 1035# possible to get a good enough coverage of different compilers / distros etc. 1036# with bazel) 1037# - some things that are considered "tests" in build.yaml-based builds are actually binaries 1038# we'd want to be able to build anyway (qps_json_worker, interop_client, interop_server, grpc_cli) 1039# so it's unclear how much make/cmake simplification we would gain by removing just some (but not all) test 1040# TODO(jtattermusch): Investigate feasibility of running portability suite with bazel. 1041tests = _exclude_unwanted_cc_tests(_extract_cc_tests(bazel_rules)) 1042 1043# Step 3: Generate the "extra metadata" for all our build targets. 1044# While the bazel rules give us most of the information we need, 1045# the legacy "build.yaml" format requires some additional fields that 1046# we cannot get just from bazel alone (we call that "extra metadata"). 1047# In this step, we basically analyze the build metadata we have from bazel 1048# and use heuristics to determine (and sometimes guess) the right 1049# extra metadata to use for each target. 1050# 1051# - For some targets (such as the public libraries, helper libraries 1052# and executables) determining the right extra metadata is hard to do 1053# automatically. For these targets, the extra metadata is supplied "manually" 1054# in form of the _BUILD_EXTRA_METADATA dictionary. That allows us to match 1055# the semantics of the legacy "build.yaml" as closely as possible. 1056# 1057# - For test binaries, it is possible to generate the "extra metadata" mostly 1058# automatically using a rule-based heuristic approach because most tests 1059# look and behave alike from the build's perspective. 1060# 1061# TODO(jtattermusch): Of course neither "_BUILD_EXTRA_METADATA" or 1062# the heuristic approach used for tests are ideal and they cannot be made 1063# to cover all possible situations (and are tailored to work with the way 1064# the grpc build currently works), but the idea was to start with something 1065# reasonably simple that matches the "build.yaml"-like semantics as closely 1066# as possible (to avoid changing too many things at once) and gradually get 1067# rid of the legacy "build.yaml"-specific fields one by one. Once that is done, 1068# only very little "extra metadata" would be needed and/or it would be trivial 1069# to generate it automatically. 1070all_extra_metadata = {} 1071all_extra_metadata.update(_BUILD_EXTRA_METADATA) 1072all_extra_metadata.update( 1073 _generate_build_extra_metadata_for_tests(tests, bazel_rules)) 1074 1075# Step 4: Generate the final metadata for all the targets. 1076# This is done by combining the bazel build metadata and the "extra metadata" 1077# we obtained in the previous step. 1078# In this step, we also perform some interesting massaging of the target metadata 1079# to end up with a result that is as similar to the legacy build.yaml data 1080# as possible. 1081# - Some targets get renamed (to match the legacy build.yaml target names) 1082# - Some intermediate libraries get elided ("expanded") to better match the set 1083# of targets provided by the legacy build.yaml build 1084# 1085# Originally the target renaming was introduced to address these concerns: 1086# - avoid changing too many things at the same time and avoid people getting 1087# confused by some well know targets suddenly being missing 1088# - Makefile/cmake and also language-specific generators rely on some build 1089# targets being called exactly the way they they are. Some of our testing 1090# scrips also invoke executables (e.g. "qps_json_driver") by their name. 1091# - The autogenerated test name from bazel includes the package path 1092# (e.g. "test_cpp_TEST_NAME"). Without renaming, the target names would 1093# end up pretty ugly (e.g. test_cpp_qps_qps_json_driver). 1094# TODO(jtattermusch): reevaluate the need for target renaming in the future. 1095# 1096# Example of a single generated target: 1097# 'grpc' : { 'language': 'c', 1098# 'public_headers': ['include/grpc/byte_buffer.h', ... ], 1099# 'headers': ['src/core/ext/filters/client_channel/client_channel.h', ... ], 1100# 'src': ['src/core/lib/surface/init.cc', ... ], 1101# 'deps': ['gpr', 'address_sorting', ...], 1102# ... } 1103all_targets_dict = _generate_build_metadata(all_extra_metadata, bazel_rules) 1104 1105# Step 5: convert the dictionary with all the targets to a dict that has 1106# the desired "build.yaml"-like layout. 1107# TODO(jtattermusch): We use the custom "build.yaml"-like layout because 1108# currently all other build systems use that format as their source of truth. 1109# In the future, we can get rid of this custom & legacy format entirely, 1110# but we would need to update the generators for other build systems 1111# at the same time. 1112# 1113# Layout of the result: 1114# { 'libs': { TARGET_DICT_FOR_LIB_XYZ, ... }, 1115# 'targets': { TARGET_DICT_FOR_BIN_XYZ, ... }, 1116# 'tests': { TARGET_DICT_FOR_TEST_XYZ, ...} } 1117build_yaml_like = _convert_to_build_yaml_like(all_targets_dict) 1118 1119# detect and report some suspicious situations we've seen before 1120_detect_and_print_issues(build_yaml_like) 1121 1122# Step 6: Store the build_autogenerated.yaml in a deterministic (=sorted) 1123# and cleaned-up form. 1124# A basic overview of the resulting "build.yaml"-like format is here: 1125# https://github.com/grpc/grpc/blob/master/templates/README.md 1126# TODO(jtattermusch): The "cleanup" function is taken from the legacy 1127# build system (which used build.yaml) and can be eventually removed. 1128build_yaml_string = build_cleaner.cleaned_build_yaml_dict_as_string( 1129 build_yaml_like) 1130with open('build_autogenerated.yaml', 'w') as file: 1131 file.write(build_yaml_string) 1132