1"""Module extension for generating third-party crates for use in bazel.""" 2 3load("@bazel_skylib//lib:structs.bzl", "structs") 4load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") 5load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 6load("//crate_universe:defs.bzl", _crate_universe_crate = "crate") 7load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest") 8load("//crate_universe/private:generate_utils.bzl", "render_config") 9load("//crate_universe/private/module_extensions:cargo_bazel_bootstrap.bzl", "get_cargo_bazel_runner") 10 11# A list of labels which may be relative (and if so, is within the repo the rule is generated in). 12# 13# If I were to write ":foo", with attr.label_list, it would evaluate to 14# "@@//:foo". However, for a tag such as deps, ":foo" should refer to 15# "@@rules_rust~crates~<crate>//:foo". 16_relative_label_list = attr.string_list 17 18_OPT_BOOL_VALUES = { 19 "auto": None, 20 "off": False, 21 "on": True, 22} 23 24def optional_bool(doc): 25 return attr.string(doc = doc, values = _OPT_BOOL_VALUES.keys(), default = "auto") 26 27def _get_or_insert(d, key, value): 28 if key not in d: 29 d[key] = value 30 return d[key] 31 32def _generate_repo_impl(repo_ctx): 33 for path, contents in repo_ctx.attr.contents.items(): 34 repo_ctx.file(path, contents) 35 36_generate_repo = repository_rule( 37 implementation = _generate_repo_impl, 38 attrs = dict( 39 contents = attr.string_dict(mandatory = True), 40 ), 41) 42 43def _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations): 44 cargo_lockfile = module_ctx.path(cfg.cargo_lockfile) 45 tag_path = module_ctx.path(cfg.name) 46 47 rendering_config = json.decode(render_config( 48 regen_command = "Run 'cargo update [--workspace]'", 49 )) 50 config_file = tag_path.get_child("config.json") 51 module_ctx.file( 52 config_file, 53 executable = False, 54 content = generate_config_file( 55 module_ctx, 56 mode = "remote", 57 annotations = annotations, 58 generate_build_scripts = cfg.generate_build_scripts, 59 supported_platform_triples = cfg.supported_platform_triples, 60 generate_target_compatible_with = True, 61 repository_name = cfg.name, 62 output_pkg = cfg.name, 63 workspace_name = cfg.name, 64 generate_binaries = cfg.generate_binaries, 65 render_config = rendering_config, 66 repository_ctx = module_ctx, 67 ), 68 ) 69 70 manifests = {module_ctx.path(m): m for m in cfg.manifests} 71 splicing_manifest = tag_path.get_child("splicing_manifest.json") 72 module_ctx.file( 73 splicing_manifest, 74 executable = False, 75 content = generate_splicing_manifest( 76 packages = {}, 77 splicing_config = "", 78 cargo_config = cfg.cargo_config, 79 manifests = {str(k): str(v) for k, v in manifests.items()}, 80 manifest_to_path = module_ctx.path, 81 ), 82 ) 83 84 splicing_output_dir = tag_path.get_child("splicing-output") 85 cargo_bazel([ 86 "splice", 87 "--output-dir", 88 splicing_output_dir, 89 "--config", 90 config_file, 91 "--splicing-manifest", 92 splicing_manifest, 93 "--cargo-lockfile", 94 cargo_lockfile, 95 ]) 96 97 # Create a lockfile, since we need to parse it to generate spoke 98 # repos. 99 lockfile_path = tag_path.get_child("lockfile.json") 100 module_ctx.file(lockfile_path, "") 101 102 cargo_bazel([ 103 "generate", 104 "--cargo-lockfile", 105 cargo_lockfile, 106 "--config", 107 config_file, 108 "--splicing-manifest", 109 splicing_manifest, 110 "--repository-dir", 111 tag_path, 112 "--metadata", 113 splicing_output_dir.get_child("metadata.json"), 114 "--repin", 115 "--lockfile", 116 lockfile_path, 117 ]) 118 119 crates_dir = tag_path.get_child(cfg.name) 120 _generate_repo( 121 name = cfg.name, 122 contents = { 123 "BUILD.bazel": module_ctx.read(crates_dir.get_child("BUILD.bazel")), 124 "defs.bzl": module_ctx.read(crates_dir.get_child("defs.bzl")), 125 }, 126 ) 127 128 contents = json.decode(module_ctx.read(lockfile_path)) 129 130 for crate in contents["crates"].values(): 131 repo = crate["repository"] 132 if repo == None: 133 continue 134 name = crate["name"] 135 version = crate["version"] 136 137 # "+" isn't valid in a repo name. 138 crate_repo_name = "{repo_name}__{name}-{version}".format( 139 repo_name = cfg.name, 140 name = name, 141 version = version.replace("+", "-"), 142 ) 143 144 build_file_content = module_ctx.read(crates_dir.get_child("BUILD.%s-%s.bazel" % (name, version))) 145 if "Http" in repo: 146 # Replicates functionality in repo_http.j2. 147 repo = repo["Http"] 148 http_archive( 149 name = crate_repo_name, 150 patch_args = repo.get("patch_args", None), 151 patch_tool = repo.get("patch_tool", None), 152 patches = repo.get("patches", None), 153 remote_patch_strip = 1, 154 sha256 = repo.get("sha256", None), 155 type = "tar.gz", 156 urls = [repo["url"]], 157 strip_prefix = "%s-%s" % (crate["name"], crate["version"]), 158 build_file_content = build_file_content, 159 ) 160 elif "Git" in repo: 161 # Replicates functionality in repo_git.j2 162 repo = repo["Git"] 163 kwargs = {} 164 for k, v in repo["commitish"].items(): 165 if k == "Rev": 166 kwargs["commit"] = v 167 else: 168 kwargs[k.lower()] = v 169 new_git_repository( 170 name = crate_repo_name, 171 init_submodules = True, 172 patch_args = repo.get("patch_args", None), 173 patch_tool = repo.get("patch_tool", None), 174 patches = repo.get("patches", None), 175 shallow_since = repo.get("shallow_since", None), 176 remote = repo["remote"], 177 build_file_content = build_file_content, 178 strip_prefix = repo.get("strip_prefix", None), 179 **kwargs 180 ) 181 else: 182 fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo)) 183 184def _crate_impl(module_ctx): 185 cargo_bazel = get_cargo_bazel_runner(module_ctx) 186 all_repos = [] 187 for mod in module_ctx.modules: 188 module_annotations = {} 189 repo_specific_annotations = {} 190 for annotation_tag in mod.tags.annotation: 191 annotation_dict = structs.to_dict(annotation_tag) 192 repositories = annotation_dict.pop("repositories") 193 crate = annotation_dict.pop("crate") 194 195 # The crate.annotation function can take in either a list or a bool. 196 # For the tag-based method, because it has type safety, we have to 197 # split it into two parameters. 198 if annotation_dict.pop("gen_all_binaries"): 199 annotation_dict["gen_binaries"] = True 200 annotation_dict["gen_build_script"] = _OPT_BOOL_VALUES[annotation_dict["gen_build_script"]] 201 annotation = _crate_universe_crate.annotation(**{ 202 k: v 203 for k, v in annotation_dict.items() 204 # Tag classes can't take in None, but the function requires None 205 # instead of the empty values in many cases. 206 # https://github.com/bazelbuild/bazel/issues/20744 207 if v != "" and v != [] and v != {} 208 }) 209 if not repositories: 210 _get_or_insert(module_annotations, crate, []).append(annotation) 211 for repo in repositories: 212 _get_or_insert( 213 _get_or_insert(repo_specific_annotations, repo, {}), 214 crate, 215 [], 216 ).append(annotation) 217 218 local_repos = [] 219 for cfg in mod.tags.from_cargo: 220 if cfg.name in local_repos: 221 fail("Defined two crate universes with the same name in the same MODULE.bazel file. Use the name tag to give them different names.") 222 elif cfg.name in all_repos: 223 fail("Defined two crate universes with the same name in different MODULE.bazel files. Either give one a different name, or use use_extension(isolate=True)") 224 225 annotations = {k: v for k, v in module_annotations.items()} 226 for crate, values in repo_specific_annotations.get(cfg.name, {}).items(): 227 _get_or_insert(annotations, crate, []).extend(values) 228 _generate_hub_and_spokes(module_ctx, cargo_bazel, cfg, annotations) 229 all_repos.append(cfg.name) 230 local_repos.append(cfg.name) 231 232 for repo in repo_specific_annotations: 233 if repo not in local_repos: 234 fail("Annotation specified for repo %s, but the module defined repositories %s" % (repo, local_repos)) 235 236_from_cargo = tag_class( 237 doc = "Generates a repo @crates from a Cargo.toml / Cargo.lock pair", 238 attrs = dict( 239 name = attr.string(doc = "The name of the repo to generate", default = "crates"), 240 cargo_lockfile = CRATES_VENDOR_ATTRS["cargo_lockfile"], 241 manifests = CRATES_VENDOR_ATTRS["manifests"], 242 cargo_config = CRATES_VENDOR_ATTRS["cargo_config"], 243 generate_binaries = CRATES_VENDOR_ATTRS["generate_binaries"], 244 generate_build_scripts = CRATES_VENDOR_ATTRS["generate_build_scripts"], 245 supported_platform_triples = CRATES_VENDOR_ATTRS["supported_platform_triples"], 246 ), 247) 248 249# This should be kept in sync with crate_universe/private/crate.bzl. 250_annotation = tag_class( 251 attrs = dict( 252 repositories = attr.string_list(doc = "A list of repository names specified from `crate.from_cargo(name=...)` that this annotation is applied to. Defaults to all repositories.", default = []), 253 crate = attr.string(doc = "The name of the crate the annotation is applied to", mandatory = True), 254 version = attr.string(doc = "The versions of the crate the annotation is applied to. Defaults to all versions.", default = "*"), 255 additive_build_file_content = attr.string(doc = "Extra contents to write to the bottom of generated BUILD files."), 256 additive_build_file = attr.label(doc = "A file containing extra contents to write to the bottom of generated BUILD files."), 257 alias_rule = attr.string(doc = "Alias rule to use instead of `native.alias()`. Overrides [render_config](#render_config)'s 'default_alias_rule'."), 258 build_script_data = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::data` attribute."), 259 build_script_tools = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::tools` attribute."), 260 build_script_data_glob = attr.string_list(doc = "A list of glob patterns to add to a crate's `cargo_build_script::data` attribute"), 261 build_script_deps = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::deps` attribute."), 262 build_script_env = attr.string_dict(doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute."), 263 build_script_proc_macro_deps = _relative_label_list(doc = "A list of labels to add to a crate's `cargo_build_script::proc_macro_deps` attribute."), 264 build_script_rundir = attr.string(doc = "An override for the build script's rundir attribute."), 265 build_script_rustc_env = attr.string_dict(doc = "Additional environment variables to set on a crate's `cargo_build_script::env` attribute."), 266 build_script_toolchains = attr.label_list(doc = "A list of labels to set on a crates's `cargo_build_script::toolchains` attribute."), 267 compile_data = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::compile_data` attribute."), 268 compile_data_glob = attr.string_list(doc = "A list of glob patterns to add to a crate's `rust_library::compile_data` attribute."), 269 crate_features = attr.string_list(doc = "A list of strings to add to a crate's `rust_library::crate_features` attribute."), 270 data = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::data` attribute."), 271 data_glob = attr.string_list(doc = "A list of glob patterns to add to a crate's `rust_library::data` attribute."), 272 deps = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::deps` attribute."), 273 extra_aliased_targets = attr.string_dict(doc = "A list of targets to add to the generated aliases in the root crate_universe repository."), 274 gen_binaries = attr.string_list(doc = "As a list, the subset of the crate's bins that should get `rust_binary` targets produced."), 275 gen_all_binaries = attr.bool(doc = "If true, generates `rust_binary` targets for all of the crates bins"), 276 disable_pipelining = attr.bool(doc = "If True, disables pipelining for library targets for this crate."), 277 gen_build_script = attr.string( 278 doc = "An authorative flag to determine whether or not to produce `cargo_build_script` targets for the current crate. Supported values are 'on', 'off', and 'auto'.", 279 values = _OPT_BOOL_VALUES.keys(), 280 default = "auto", 281 ), 282 patch_args = attr.string_list(doc = "The `patch_args` attribute of a Bazel repository rule. See [http_archive.patch_args](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_args)"), 283 patch_tool = attr.string(doc = "The `patch_tool` attribute of a Bazel repository rule. See [http_archive.patch_tool](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patch_tool)"), 284 patches = attr.label_list(doc = "The `patches` attribute of a Bazel repository rule. See [http_archive.patches](https://docs.bazel.build/versions/main/repo/http.html#http_archive-patches)"), 285 proc_macro_deps = _relative_label_list(doc = "A list of labels to add to a crate's `rust_library::proc_macro_deps` attribute."), 286 rustc_env = attr.string_dict(doc = "Additional variables to set on a crate's `rust_library::rustc_env` attribute."), 287 rustc_env_files = _relative_label_list(doc = "A list of labels to set on a crate's `rust_library::rustc_env_files` attribute."), 288 rustc_flags = attr.string_list(doc = "A list of strings to set on a crate's `rust_library::rustc_flags` attribute."), 289 shallow_since = attr.string(doc = "An optional timestamp used for crates originating from a git repository instead of a crate registry. This flag optimizes fetching the source code."), 290 ), 291) 292 293crate = module_extension( 294 implementation = _crate_impl, 295 tag_classes = dict( 296 from_cargo = _from_cargo, 297 annotation = _annotation, 298 ), 299) 300