1#!/usr/bin/env python3 2 3# Copyright 2022 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import collections 19from doctest import SKIP 20import multiprocessing 21import os 22import re 23import sys 24 25import run_buildozer 26 27# find our home 28ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) 29os.chdir(ROOT) 30 31vendors = collections.defaultdict(list) 32scores = collections.defaultdict(int) 33avoidness = collections.defaultdict(int) 34consumes = {} 35no_update = set() 36buildozer_commands = [] 37original_deps = {} 38original_external_deps = {} 39skip_headers = collections.defaultdict(set) 40 41# TODO(ctiller): ideally we wouldn't hardcode a bunch of paths here. 42# We can likely parse out BUILD files from dependencies to generate this index. 43EXTERNAL_DEPS = { 44 "absl/algorithm/container.h": "absl/algorithm:container", 45 "absl/base/attributes.h": "absl/base:core_headers", 46 "absl/base/call_once.h": "absl/base", 47 "absl/base/config.h": "absl/base:config", 48 # TODO(ctiller) remove this 49 "absl/base/internal/endian.h": "absl/base:endian", 50 "absl/base/thread_annotations.h": "absl/base:core_headers", 51 "absl/container/flat_hash_map.h": "absl/container:flat_hash_map", 52 "absl/container/flat_hash_set.h": "absl/container:flat_hash_set", 53 "absl/container/inlined_vector.h": "absl/container:inlined_vector", 54 "absl/cleanup/cleanup.h": "absl/cleanup", 55 "absl/debugging/failure_signal_handler.h": ( 56 "absl/debugging:failure_signal_handler" 57 ), 58 "absl/debugging/stacktrace.h": "absl/debugging:stacktrace", 59 "absl/debugging/symbolize.h": "absl/debugging:symbolize", 60 "absl/flags/flag.h": "absl/flags:flag", 61 "absl/flags/marshalling.h": "absl/flags:marshalling", 62 "absl/flags/parse.h": "absl/flags:parse", 63 "absl/functional/any_invocable.h": "absl/functional:any_invocable", 64 "absl/functional/bind_front.h": "absl/functional:bind_front", 65 "absl/functional/function_ref.h": "absl/functional:function_ref", 66 "absl/hash/hash.h": "absl/hash", 67 "absl/log/check.h": "absl/log:check", 68 "absl/log/log.h": "absl/log", 69 "absl/memory/memory.h": "absl/memory", 70 "absl/meta/type_traits.h": "absl/meta:type_traits", 71 "absl/numeric/int128.h": "absl/numeric:int128", 72 "absl/random/random.h": "absl/random", 73 "absl/random/bit_gen_ref.h": "absl/random:bit_gen_ref", 74 "absl/random/mocking_bit_gen.h": "absl/random:mocking_bit_gen", 75 "absl/random/distributions.h": "absl/random:distributions", 76 "absl/random/uniform_int_distribution.h": "absl/random:distributions", 77 "absl/status/status.h": "absl/status", 78 "absl/status/statusor.h": "absl/status:statusor", 79 "absl/strings/ascii.h": "absl/strings", 80 "absl/strings/cord.h": "absl/strings:cord", 81 "absl/strings/escaping.h": "absl/strings", 82 "absl/strings/match.h": "absl/strings", 83 "absl/strings/numbers.h": "absl/strings", 84 "absl/strings/str_cat.h": "absl/strings", 85 "absl/strings/str_format.h": "absl/strings:str_format", 86 "absl/strings/str_join.h": "absl/strings", 87 "absl/strings/str_replace.h": "absl/strings", 88 "absl/strings/str_split.h": "absl/strings", 89 "absl/strings/string_view.h": "absl/strings", 90 "absl/strings/strip.h": "absl/strings", 91 "absl/strings/substitute.h": "absl/strings", 92 "absl/synchronization/mutex.h": "absl/synchronization", 93 "absl/synchronization/notification.h": "absl/synchronization", 94 "absl/time/clock.h": "absl/time", 95 "absl/time/time.h": "absl/time", 96 "absl/types/optional.h": "absl/types:optional", 97 "absl/types/span.h": "absl/types:span", 98 "absl/types/variant.h": "absl/types:variant", 99 "absl/utility/utility.h": "absl/utility", 100 "benchmark/benchmark.h": "benchmark", 101 "address_sorting/address_sorting.h": "address_sorting", 102 "google/cloud/opentelemetry/resource_detector.h": "google_cloud_cpp:opentelemetry", 103 "opentelemetry/common/attribute_value.h": "otel/api", 104 "opentelemetry/common/key_value_iterable.h": "otel/api", 105 "opentelemetry/nostd/function_ref.h": "otel/api", 106 "opentelemetry/nostd/string_view.h": "otel/api", 107 "opentelemetry/context/context.h": "otel/api", 108 "opentelemetry/metrics/meter.h": "otel/api", 109 "opentelemetry/metrics/meter_provider.h": "otel/api", 110 "opentelemetry/metrics/provider.h": "otel/api", 111 "opentelemetry/metrics/sync_instruments.h": "otel/api", 112 "opentelemetry/nostd/shared_ptr.h": "otel/api", 113 "opentelemetry/nostd/unique_ptr.h": "otel/api", 114 "opentelemetry/sdk/metrics/meter_provider.h": "otel/sdk/src/metrics", 115 "opentelemetry/sdk/common/attribute_utils.h": "otel/sdk:headers", 116 "opentelemetry/sdk/resource/resource.h": "otel/sdk:headers", 117 "opentelemetry/sdk/resource/resource_detector.h": "otel/sdk:headers", 118 "opentelemetry/sdk/resource/semantic_conventions.h": "otel/sdk:headers", 119 "ares.h": "cares", 120 "fuzztest/fuzztest.h": ["fuzztest", "fuzztest_main"], 121 "google/api/monitored_resource.pb.h": ( 122 "google/api:monitored_resource_cc_proto" 123 ), 124 "google/devtools/cloudtrace/v2/tracing.grpc.pb.h": ( 125 "googleapis_trace_grpc_service" 126 ), 127 "google/logging/v2/logging.grpc.pb.h": "googleapis_logging_grpc_service", 128 "google/logging/v2/logging.pb.h": "googleapis_logging_cc_proto", 129 "google/logging/v2/log_entry.pb.h": "googleapis_logging_cc_proto", 130 "google/monitoring/v3/metric_service.grpc.pb.h": ( 131 "googleapis_monitoring_grpc_service" 132 ), 133 "gmock/gmock.h": "gtest", 134 "gtest/gtest.h": "gtest", 135 "opencensus/exporters/stats/stackdriver/stackdriver_exporter.h": ( 136 "opencensus-stats-stackdriver_exporter" 137 ), 138 "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h": ( 139 "opencensus-trace-stackdriver_exporter" 140 ), 141 "opencensus/trace/context_util.h": "opencensus-trace-context_util", 142 "opencensus/trace/propagation/grpc_trace_bin.h": ( 143 "opencensus-trace-propagation" 144 ), 145 "opencensus/tags/context_util.h": "opencensus-tags-context_util", 146 "opencensus/trace/span_context.h": "opencensus-trace-span_context", 147 "openssl/base.h": "libssl", 148 "openssl/bio.h": "libssl", 149 "openssl/bn.h": "libcrypto", 150 "openssl/buffer.h": "libcrypto", 151 "openssl/crypto.h": "libcrypto", 152 "openssl/digest.h": "libssl", 153 "openssl/engine.h": "libcrypto", 154 "openssl/err.h": "libcrypto", 155 "openssl/evp.h": "libcrypto", 156 "openssl/hmac.h": "libcrypto", 157 "openssl/mem.h": "libcrypto", 158 "openssl/param_build.h": "libcrypto", 159 "openssl/pem.h": "libcrypto", 160 "openssl/rsa.h": "libcrypto", 161 "openssl/sha.h": "libcrypto", 162 "openssl/ssl.h": "libssl", 163 "openssl/tls1.h": "libssl", 164 "openssl/x509.h": "libcrypto", 165 "openssl/x509v3.h": "libcrypto", 166 "re2/re2.h": "re2", 167 "upb/base/status.hpp": "upb_base_lib", 168 "upb/base/string_view.h": "upb_base_lib", 169 "upb/message/map.h": "upb_message_lib", 170 "upb/reflection/def.h": "upb_reflection", 171 "upb/json/encode.h": "upb_json_lib", 172 "upb/mem/arena.h": "upb_mem_lib", 173 "upb/mem/arena.hpp": "upb_mem_lib", 174 "upb/text/encode.h": "upb_textformat_lib", 175 "upb/reflection/def.hpp": "upb_reflection", 176 "xxhash.h": "xxhash", 177 "zlib.h": "madler_zlib", 178} 179 180INTERNAL_DEPS = { 181 "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h": ( 182 "//test/core/event_engine/fuzzing_event_engine" 183 ), 184 "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h": "//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto", 185 "test/core/experiments/test_experiments.h": "//test/core/experiments:test_experiments_lib", 186 "google/api/expr/v1alpha1/syntax.upb.h": "google_api_expr_v1alpha1_syntax_upb", 187 "google/rpc/status.upb.h": "google_rpc_status_upb", 188 "google/protobuf/any.upb.h": "protobuf_any_upb", 189 "google/protobuf/duration.upb.h": "protobuf_duration_upb", 190 "google/protobuf/struct.upb.h": "protobuf_struct_upb", 191 "google/protobuf/timestamp.upb.h": "protobuf_timestamp_upb", 192 "google/protobuf/wrappers.upb.h": "protobuf_wrappers_upb", 193 "grpc/status.h": "grpc_public_hdrs", 194 "src/proto/grpc/channelz/channelz.grpc.pb.h": ( 195 "//src/proto/grpc/channelz:channelz_proto" 196 ), 197 "src/proto/grpc/core/stats.pb.h": "//src/proto/grpc/core:stats_proto", 198 "src/proto/grpc/health/v1/health.upb.h": "grpc_health_upb", 199 "src/proto/grpc/lb/v1/load_reporter.grpc.pb.h": ( 200 "//src/proto/grpc/lb/v1:load_reporter_proto" 201 ), 202 "src/proto/grpc/lb/v1/load_balancer.upb.h": "grpc_lb_upb", 203 "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h": ( 204 "//src/proto/grpc/reflection/v1alpha:reflection_proto" 205 ), 206 "src/proto/grpc/gcp/transport_security_common.upb.h": "alts_upb", 207 "src/proto/grpc/gcp/handshaker.upb.h": "alts_upb", 208 "src/proto/grpc/gcp/altscontext.upb.h": "alts_upb", 209 "src/proto/grpc/lookup/v1/rls.upb.h": "rls_upb", 210 "src/proto/grpc/lookup/v1/rls_config.upb.h": "rls_config_upb", 211 "src/proto/grpc/lookup/v1/rls_config.upbdefs.h": "rls_config_upbdefs", 212 "src/proto/grpc/testing/xds/v3/csds.grpc.pb.h": ( 213 "//src/proto/grpc/testing/xds/v3:csds_proto" 214 ), 215 "xds/data/orca/v3/orca_load_report.upb.h": "xds_orca_upb", 216 "xds/service/orca/v3/orca.upb.h": "xds_orca_service_upb", 217 "xds/type/v3/typed_struct.upb.h": "xds_type_upb", 218} 219 220 221class FakeSelects: 222 def config_setting_group(self, **kwargs): 223 pass 224 225 226num_cc_libraries = 0 227num_opted_out_cc_libraries = 0 228parsing_path = None 229 230 231# Convert the source or header target to a relative path. 232def _get_filename(name, parsing_path): 233 filename = "%s%s" % ( 234 ( 235 parsing_path + "/" 236 if (parsing_path and not name.startswith("//")) 237 else "" 238 ), 239 name, 240 ) 241 filename = filename.replace("//:", "") 242 filename = filename.replace("//src/core:", "src/core/") 243 filename = filename.replace( 244 "//src/cpp/ext/filters/census:", "src/cpp/ext/filters/census/" 245 ) 246 return filename 247 248 249def grpc_cc_library( 250 name, 251 hdrs=[], 252 public_hdrs=[], 253 srcs=[], 254 select_deps=None, 255 tags=[], 256 deps=[], 257 external_deps=[], 258 proto=None, 259 **kwargs, 260): 261 global args 262 global num_cc_libraries 263 global num_opted_out_cc_libraries 264 global parsing_path 265 assert parsing_path is not None 266 try: 267 name = "//%s:%s" % (parsing_path, name) 268 num_cc_libraries += 1 269 if select_deps or "nofixdeps" in tags: 270 if args.whats_left and not select_deps and "nofixdeps" not in tags: 271 num_opted_out_cc_libraries += 1 272 print("Not opted in: {}".format(name)) 273 no_update.add(name) 274 scores[name] = len(public_hdrs + hdrs) 275 # avoid_dep is the internal way of saying prefer something else 276 # we add grpc_avoid_dep to allow internal grpc-only stuff to avoid each 277 # other, whilst not biasing dependent projects 278 if "avoid_dep" in tags or "grpc_avoid_dep" in tags: 279 avoidness[name] += 10 280 if proto: 281 proto_hdr = "%s%s" % ( 282 (parsing_path + "/" if parsing_path else ""), 283 proto.replace(".proto", ".pb.h"), 284 ) 285 skip_headers[name].add(proto_hdr) 286 287 for hdr in hdrs + public_hdrs: 288 vendors[_get_filename(hdr, parsing_path)].append(name) 289 inc = set() 290 original_deps[name] = frozenset(deps) 291 original_external_deps[name] = frozenset(external_deps) 292 for src in hdrs + public_hdrs + srcs: 293 for line in open(_get_filename(src, parsing_path)): 294 m = re.search(r"^#include <(.*)>", line) 295 if m: 296 inc.add(m.group(1)) 297 m = re.search(r'^#include "(.*)"', line) 298 if m: 299 inc.add(m.group(1)) 300 consumes[name] = list(inc) 301 except: 302 print("Error while parsing ", name) 303 raise 304 305 306def grpc_proto_library(name, srcs, **kwargs): 307 global parsing_path 308 assert parsing_path is not None 309 name = "//%s:%s" % (parsing_path, name) 310 for src in srcs: 311 proto_hdr = src.replace(".proto", ".pb.h") 312 vendors[_get_filename(proto_hdr, parsing_path)].append(name) 313 314 315def buildozer(cmd, target): 316 buildozer_commands.append("%s|%s" % (cmd, target)) 317 318 319def buildozer_set_list(name, values, target, via=""): 320 if not values: 321 buildozer("remove %s" % name, target) 322 return 323 adjust = via if via else name 324 buildozer( 325 "set %s %s" % (adjust, " ".join('"%s"' % s for s in values)), target 326 ) 327 if via: 328 buildozer("remove %s" % name, target) 329 buildozer("rename %s %s" % (via, name), target) 330 331 332def score_edit_distance(proposed, existing): 333 """Score a proposed change primarily by edit distance""" 334 sum = 0 335 for p in proposed: 336 if p not in existing: 337 sum += 1 338 for e in existing: 339 if e not in proposed: 340 sum += 1 341 return sum 342 343 344def total_score(proposal): 345 return sum(scores[dep] for dep in proposal) 346 347 348def total_avoidness(proposal): 349 return sum(avoidness[dep] for dep in proposal) 350 351 352def score_list_size(proposed, existing): 353 """Score a proposed change primarily by number of dependencies""" 354 return len(proposed) 355 356 357def score_best(proposed, existing): 358 """Score a proposed change primarily by dependency score""" 359 return 0 360 361 362SCORERS = { 363 "edit_distance": score_edit_distance, 364 "list_size": score_list_size, 365 "best": score_best, 366} 367 368parser = argparse.ArgumentParser(description="Fix build dependencies") 369parser.add_argument("targets", nargs="+", help="targets to fix") 370parser.add_argument( 371 "--score", 372 type=str, 373 default="edit_distance", 374 help="scoring function to use: one of " + ", ".join(SCORERS.keys()), 375) 376parser.add_argument( 377 "--whats_left", 378 action="store_true", 379 default=False, 380 help="show what is left to opt in", 381) 382parser.add_argument( 383 "--explain", 384 action="store_true", 385 default=False, 386 help="try to explain some decisions", 387) 388parser.add_argument( 389 "--why", 390 type=str, 391 default=None, 392 help="with --explain, target why a given dependency is needed", 393) 394args = parser.parse_args() 395 396for dirname in [ 397 "", 398 "src/core", 399 "src/cpp/ext/gcp", 400 "src/cpp/ext/csm", 401 "src/cpp/ext/otel", 402 "test/core/util", 403 "test/core/call", 404 "test/core/call/yodel", 405 "test/core/client_channel", 406 "test/core/experiments", 407 "test/core/load_balancing", 408 "test/core/util", 409 "test/core/test_util", 410 "test/core/end2end", 411 "test/core/event_engine", 412 "test/core/filters", 413 "test/core/promise", 414 "test/core/resource_quota", 415 "test/core/transport/chaotic_good", 416 "test/core/transport/test_suite", 417 "test/core/transport", 418 "fuzztest", 419 "fuzztest/core/channel", 420 "fuzztest/core/transport/chttp2", 421]: 422 parsing_path = dirname 423 exec( 424 open("%sBUILD" % (dirname + "/" if dirname else ""), "r").read(), 425 { 426 "load": lambda filename, *args: None, 427 "licenses": lambda licenses: None, 428 "package": lambda **kwargs: None, 429 "exports_files": lambda files, visibility=None: None, 430 "bool_flag": lambda **kwargs: None, 431 "config_setting": lambda **kwargs: None, 432 "selects": FakeSelects(), 433 "python_config_settings": lambda **kwargs: None, 434 "grpc_cc_benchmark": grpc_cc_library, 435 "grpc_cc_binary": grpc_cc_library, 436 "grpc_cc_library": grpc_cc_library, 437 "grpc_cc_test": grpc_cc_library, 438 "grpc_cc_benchmark": grpc_cc_library, 439 "grpc_core_end2end_test": lambda **kwargs: None, 440 "grpc_filegroup": lambda **kwargs: None, 441 "grpc_transport_test": lambda **kwargs: None, 442 "grpc_yodel_test": lambda **kwargs: None, 443 "grpc_yodel_simple_test": lambda **kwargs: None, 444 "grpc_fuzzer": grpc_cc_library, 445 "grpc_fuzz_test": grpc_cc_library, 446 "grpc_proto_fuzzer": grpc_cc_library, 447 "grpc_proto_library": grpc_proto_library, 448 "grpc_internal_proto_library": grpc_proto_library, 449 "grpc_cc_proto_library": grpc_cc_library, 450 "select": lambda d: d["//conditions:default"], 451 "glob": lambda files, **kwargs: None, 452 "grpc_end2end_tests": lambda: None, 453 "grpc_upb_proto_library": lambda name, **kwargs: None, 454 "grpc_upb_proto_reflection_library": lambda name, **kwargs: None, 455 "grpc_generate_one_off_targets": lambda: None, 456 "grpc_generate_one_off_internal_targets": lambda: None, 457 "grpc_package": lambda **kwargs: None, 458 "filegroup": lambda name, **kwargs: None, 459 "sh_library": lambda name, **kwargs: None, 460 "platform": lambda name, **kwargs: None, 461 "grpc_clang_cl_settings": lambda **kwargs: None, 462 "grpc_benchmark_args": lambda **kwargs: [], 463 "LARGE_MACHINE": 1, 464 "HISTORY": 1, 465 }, 466 {}, 467 ) 468 parsing_path = None 469 470if args.whats_left: 471 print( 472 "{}/{} libraries are opted in".format( 473 num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries 474 ) 475 ) 476 477 478def make_relative_path(dep, lib): 479 if lib is None: 480 return dep 481 lib_path = lib[: lib.rfind(":") + 1] 482 if dep.startswith(lib_path): 483 return dep[len(lib_path) :] 484 return dep 485 486 487if args.whats_left: 488 print( 489 "{}/{} libraries are opted in".format( 490 num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries 491 ) 492 ) 493 494 495# Keeps track of all possible sets of dependencies that could satisfy the 496# problem. (models the list monad in Haskell!) 497class Choices: 498 def __init__(self, library, substitutions): 499 self.library = library 500 self.to_add = [] 501 self.to_remove = [] 502 self.substitutions = substitutions 503 504 def add_one_of(self, choices, trigger): 505 if not choices: 506 return 507 choices = sum( 508 [self.apply_substitutions(choice) for choice in choices], [] 509 ) 510 if args.explain and (args.why is None or args.why in choices): 511 print( 512 "{}: Adding one of {} for {}".format( 513 self.library, choices, trigger 514 ) 515 ) 516 self.to_add.append( 517 tuple( 518 make_relative_path(choice, self.library) for choice in choices 519 ) 520 ) 521 522 def add(self, choice, trigger): 523 self.add_one_of([choice], trigger) 524 525 def remove(self, remove): 526 for remove in self.apply_substitutions(remove): 527 self.to_remove.append(make_relative_path(remove, self.library)) 528 529 def apply_substitutions(self, dep): 530 if dep in self.substitutions: 531 return self.substitutions[dep] 532 return [dep] 533 534 def best(self, scorer): 535 choices = set() 536 choices.add(frozenset()) 537 538 for add in sorted(set(self.to_add), key=lambda x: (len(x), x)): 539 new_choices = set() 540 for append_choice in add: 541 for choice in choices: 542 new_choices.add(choice.union([append_choice])) 543 choices = new_choices 544 for remove in sorted(set(self.to_remove)): 545 new_choices = set() 546 for choice in choices: 547 new_choices.add(choice.difference([remove])) 548 choices = new_choices 549 550 best = None 551 552 def final_scorer(x): 553 return (total_avoidness(x), scorer(x), total_score(x)) 554 555 for choice in choices: 556 if best is None or final_scorer(choice) < final_scorer(best): 557 best = choice 558 return best 559 560 561def make_library(library): 562 error = False 563 hdrs = sorted(consumes[library]) 564 # we need a little trickery here since grpc_base has channel.cc, which calls grpc_init 565 # which is in grpc, which is illegal but hard to change 566 # once EventEngine lands we can clean this up 567 deps = Choices( 568 library, 569 ( 570 {"//:grpc_base": ["//:grpc", "//:grpc_unsecure"]} 571 if library.startswith("//test/") 572 else {} 573 ), 574 ) 575 external_deps = Choices(None, {}) 576 for hdr in hdrs: 577 if hdr in skip_headers[library]: 578 continue 579 580 if hdr == "systemd/sd-daemon.h": 581 continue 582 583 if hdr == "src/core/lib/profiling/stap_probes.h": 584 continue 585 586 if hdr.startswith("src/libfuzzer/"): 587 continue 588 589 if hdr == "grpc/grpc.h" and library.startswith("//test:"): 590 # not the root build including grpc.h ==> //:grpc 591 deps.add_one_of(["//:grpc", "//:grpc_unsecure"], hdr) 592 continue 593 594 if hdr in INTERNAL_DEPS: 595 dep = INTERNAL_DEPS[hdr] 596 if isinstance(dep, list): 597 for d in dep: 598 deps.add(d, hdr) 599 else: 600 if not ("//" in dep): 601 dep = "//:" + dep 602 deps.add(dep, hdr) 603 continue 604 605 if hdr in vendors: 606 deps.add_one_of(vendors[hdr], hdr) 607 continue 608 609 if "include/" + hdr in vendors: 610 deps.add_one_of(vendors["include/" + hdr], hdr) 611 continue 612 613 if "." not in hdr: 614 # assume a c++ system include 615 continue 616 617 if hdr in EXTERNAL_DEPS: 618 if isinstance(EXTERNAL_DEPS[hdr], list): 619 for dep in EXTERNAL_DEPS[hdr]: 620 external_deps.add(dep, hdr) 621 else: 622 external_deps.add(EXTERNAL_DEPS[hdr], hdr) 623 continue 624 625 if hdr.startswith("opencensus/"): 626 trail = hdr[len("opencensus/") :] 627 trail = trail[: trail.find("/")] 628 external_deps.add("opencensus-" + trail, hdr) 629 continue 630 631 if hdr.startswith("envoy/"): 632 path, file = os.path.split(hdr) 633 file = file.split(".") 634 path = path.split("/") 635 dep = "_".join(path[:-1] + [file[1]]) 636 deps.add(dep, hdr) 637 continue 638 639 if hdr.startswith("google/protobuf/") and not hdr.endswith(".upb.h"): 640 external_deps.add("protobuf_headers", hdr) 641 continue 642 643 if "/" not in hdr: 644 # assume a system include 645 continue 646 647 is_sys_include = False 648 for sys_path in [ 649 "sys", 650 "arpa", 651 "gperftools", 652 "netinet", 653 "linux", 654 "android", 655 "mach", 656 "net", 657 "CoreFoundation", 658 ]: 659 if hdr.startswith(sys_path + "/"): 660 is_sys_include = True 661 break 662 if is_sys_include: 663 # assume a system include 664 continue 665 666 print( 667 "# ERROR: can't categorize header: %s used by %s" % (hdr, library) 668 ) 669 error = True 670 671 deps.remove(library) 672 673 deps = sorted( 674 deps.best(lambda x: SCORERS[args.score](x, original_deps[library])) 675 ) 676 external_deps = sorted( 677 external_deps.best( 678 lambda x: SCORERS[args.score](x, original_external_deps[library]) 679 ) 680 ) 681 682 return (library, error, deps, external_deps) 683 684 685def matches_target(library, target): 686 if not target.startswith("//"): 687 if "/" in target: 688 target = "//" + target 689 else: 690 target = "//:" + target 691 if target == "..." or target == "//...": 692 return True 693 if target.endswith("/..."): 694 return library.startswith(target[:-4]) 695 return library == target 696 697 698def main() -> None: 699 update_libraries = [] 700 for library in sorted(consumes.keys()): 701 if library in no_update: 702 continue 703 for target in args.targets: 704 if matches_target(library, target): 705 update_libraries.append(library) 706 break 707 with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as p: 708 updated_libraries = p.map(make_library, update_libraries, 1) 709 710 error = False 711 for library, lib_error, deps, external_deps in updated_libraries: 712 if lib_error: 713 error = True 714 continue 715 buildozer_set_list("external_deps", external_deps, library, via="deps") 716 buildozer_set_list("deps", deps, library) 717 718 run_buildozer.run_buildozer(buildozer_commands) 719 720 if error: 721 sys.exit(1) 722 723 724if __name__ == "__main__": 725 main() 726