1# Protocol Buffers - Google's data interchange format 2# Copyright 2024 Google Inc. All rights reserved. 3# 4# Use of this source code is governed by a BSD-style 5# license that can be found in the LICENSE file or at 6# https://developers.google.com/open-source/licenses/bsd 7# 8"""Definition of proto_common module, together with bazel providers for proto rules.""" 9 10load("@proto_bazel_features//:features.bzl", "bazel_features") 11load("//bazel/common:proto_lang_toolchain_info.bzl", "ProtoLangToolchainInfo") 12load("//bazel/private:native.bzl", "native_proto_common") 13load("//bazel/private:toolchain_helpers.bzl", "toolchains") 14 15def _import_virtual_proto_path(path): 16 """Imports all paths for virtual imports. 17 18 They're of the form: 19 'bazel-out/k8-fastbuild/bin/external/foo/e/_virtual_imports/e' or 20 'bazel-out/foo/k8-fastbuild/bin/e/_virtual_imports/e'""" 21 if path.count("/") > 4: 22 return "-I%s" % path 23 return None 24 25def _import_repo_proto_path(path): 26 """Imports all paths for generated files in external repositories. 27 28 They are of the form: 29 'bazel-out/k8-fastbuild/bin/external/foo' or 30 'bazel-out/foo/k8-fastbuild/bin'""" 31 path_count = path.count("/") 32 if path_count > 2 and path_count <= 4: 33 return "-I%s" % path 34 return None 35 36def _import_main_output_proto_path(path): 37 """Imports all paths for generated files or source files in external repositories. 38 39 They're of the form: 40 'bazel-out/k8-fastbuild/bin' 41 'external/foo' 42 '../foo' 43 """ 44 if path.count("/") <= 2 and path != ".": 45 return "-I%s" % path 46 return None 47 48def _remove_repo(file): 49 """Removes `../repo/` prefix from path, e.g. `../repo/package/path -> package/path`""" 50 short_path = file.short_path 51 workspace_root = file.owner.workspace_root 52 if workspace_root: 53 if workspace_root.startswith("external/"): 54 workspace_root = "../" + workspace_root.removeprefix("external/") 55 return short_path.removeprefix(workspace_root + "/") 56 return short_path 57 58def _get_import_path(proto_file): 59 """Returns the import path of a .proto file 60 61 This is the path as used for the file that can be used in an `import` statement in another 62 .proto file. 63 64 Args: 65 proto_file: (File) The .proto file 66 67 Returns: 68 (str) import path 69 """ 70 repo_path = _remove_repo(proto_file) 71 index = repo_path.find("_virtual_imports/") 72 if index >= 0: 73 index = repo_path.find("/", index + len("_virtual_imports/")) 74 repo_path = repo_path[index + 1:] 75 return repo_path 76 77def _output_directory(proto_info, root): 78 proto_source_root = proto_info.proto_source_root 79 if proto_source_root.startswith(root.path): 80 #TODO: remove this branch when bin_dir is removed from proto_source_root 81 proto_source_root = proto_source_root.removeprefix(root.path).removeprefix("/") 82 83 if proto_source_root == "" or proto_source_root == ".": 84 return root.path 85 86 return root.path + "/" + proto_source_root 87 88def _check_collocated(label, proto_info, proto_lang_toolchain_info): 89 """Checks if lang_proto_library is collocated with proto_library. 90 91 Exceptions are allowed by an allowlist defined on `proto_lang_toolchain` and 92 on an allowlist defined on `proto_library`'s `allow_exports` attribute. 93 94 If checks are not successful the function fails. 95 96 Args: 97 label: (Label) The label of lang_proto_library 98 proto_info: (ProtoInfo) The ProtoInfo from the proto_library dependency. 99 proto_lang_toolchain_info: (ProtoLangToolchainInfo) The proto lang toolchain info. 100 Obtained from a `proto_lang_toolchain` target. 101 """ 102 _PackageSpecificationInfo = bazel_features.globals.PackageSpecificationInfo 103 if not _PackageSpecificationInfo: 104 if proto_lang_toolchain_info.allowlist_different_package or getattr(proto_info, "allow_exports", None): 105 fail("Allowlist checks not supported before Bazel 6.4.0") 106 return 107 108 if (proto_info.direct_descriptor_set.owner.package != label.package and 109 proto_lang_toolchain_info.allowlist_different_package): 110 if not proto_lang_toolchain_info.allowlist_different_package[_PackageSpecificationInfo].contains(label): 111 fail(("lang_proto_library '%s' may only be created in the same package " + 112 "as proto_library '%s'") % (label, proto_info.direct_descriptor_set.owner)) 113 if (proto_info.direct_descriptor_set.owner.package != label.package and 114 hasattr(proto_info, "allow_exports")): 115 if not proto_info.allow_exports[_PackageSpecificationInfo].contains(label): 116 fail(("lang_proto_library '%s' may only be created in the same package " + 117 "as proto_library '%s'") % (label, proto_info.direct_descriptor_set.owner)) 118 119def _compile( 120 actions, 121 proto_info, 122 proto_lang_toolchain_info, 123 generated_files, 124 plugin_output = None, 125 additional_args = None, 126 additional_tools = [], 127 additional_inputs = depset(), 128 additional_proto_lang_toolchain_info = None, 129 resource_set = None, 130 experimental_exec_group = None, 131 experimental_progress_message = None, 132 experimental_output_files = "legacy"): 133 """Creates proto compile action for compiling *.proto files to language specific sources. 134 135 Args: 136 actions: (ActionFactory) Obtained by ctx.actions, used to register the actions. 137 proto_info: (ProtoInfo) The ProtoInfo from proto_library to generate the sources for. 138 proto_lang_toolchain_info: (ProtoLangToolchainInfo) The proto lang toolchain info. 139 Obtained from a `proto_lang_toolchain` target or constructed ad-hoc.. 140 generated_files: (list[File]) The output files generated by the proto compiler. 141 Callee needs to declare files using `ctx.actions.declare_file`. 142 See also: `proto_common.declare_generated_files`. 143 plugin_output: (File|str) Deprecated: Set `proto_lang_toolchain.output_files` 144 and remove the parameter. 145 For backwards compatibility, when the proto_lang_toolchain isn't updated 146 the value is used. 147 additional_args: (Args) Additional arguments to add to the action. 148 Accepts a ctx.actions.args() object that is added at the beginning 149 of the command line. 150 additional_tools: (list[File]) Additional tools to add to the action. 151 additional_inputs: (Depset[File]) Additional input files to add to the action. 152 resource_set: (func) A callback function that is passed to the created action. 153 See `ctx.actions.run`, `resource_set` parameter for full definition of 154 the callback. 155 experimental_exec_group: (str) Sets `exec_group` on proto compile action. 156 Avoid using this parameter. 157 experimental_progress_message: Overrides progress_message from the toolchain. 158 Don't use this parameter. It's only intended for the transition. 159 experimental_output_files: (str) Overwrites output_files from the toolchain. 160 Don't use this parameter. It's only intended for the transition. 161 """ 162 if type(generated_files) != type([]): 163 fail("generated_files is expected to be a list of Files") 164 if not generated_files: 165 return # nothing to do 166 if experimental_output_files not in ["single", "multiple", "legacy"]: 167 fail('experimental_output_files expected to be one of ["single", "multiple", "legacy"]') 168 169 args = actions.args() 170 args.use_param_file(param_file_arg = "@%s") 171 args.set_param_file_format("multiline") 172 tools = list(additional_tools) 173 174 if experimental_output_files != "legacy": 175 output_files = experimental_output_files 176 else: 177 output_files = getattr(proto_lang_toolchain_info, "output_files", "legacy") 178 if output_files != "legacy": 179 if proto_lang_toolchain_info.out_replacement_format_flag: 180 if output_files == "single": 181 if len(generated_files) > 1: 182 fail("generated_files only expected a single file") 183 plugin_output = generated_files[0] 184 else: 185 plugin_output = _output_directory(proto_info, generated_files[0].root) 186 187 if plugin_output: 188 args.add(plugin_output, format = proto_lang_toolchain_info.out_replacement_format_flag) 189 if proto_lang_toolchain_info.plugin: 190 tools.append(proto_lang_toolchain_info.plugin) 191 args.add(proto_lang_toolchain_info.plugin.executable, format = proto_lang_toolchain_info.plugin_format_flag) 192 193 # Protoc searches for .protos -I paths in order they are given and then 194 # uses the path within the directory as the package. 195 # This requires ordering the paths from most specific (longest) to least 196 # specific ones, so that no path in the list is a prefix of any of the 197 # following paths in the list. 198 # For example: 'bazel-out/k8-fastbuild/bin/external/foo' needs to be listed 199 # before 'bazel-out/k8-fastbuild/bin'. If not, protoc will discover file under 200 # the shorter path and use 'external/foo/...' as its package path. 201 args.add_all(proto_info.transitive_proto_path, map_each = _import_virtual_proto_path) 202 args.add_all(proto_info.transitive_proto_path, map_each = _import_repo_proto_path) 203 args.add_all(proto_info.transitive_proto_path, map_each = _import_main_output_proto_path) 204 args.add("-I.") # Needs to come last 205 206 args.add_all(proto_lang_toolchain_info.protoc_opts) 207 208 args.add_all(proto_info.direct_sources) 209 210 if additional_args: 211 additional_args.use_param_file(param_file_arg = "@%s") 212 additional_args.set_param_file_format("multiline") 213 214 actions.run( 215 mnemonic = proto_lang_toolchain_info.mnemonic, 216 progress_message = experimental_progress_message if experimental_progress_message else proto_lang_toolchain_info.progress_message, 217 executable = proto_lang_toolchain_info.proto_compiler, 218 arguments = [args, additional_args] if additional_args else [args], 219 inputs = depset(transitive = [proto_info.transitive_sources, additional_inputs]), 220 outputs = generated_files, 221 tools = tools, 222 use_default_shell_env = True, 223 resource_set = resource_set, 224 exec_group = experimental_exec_group, 225 toolchain = _toolchain_type(proto_lang_toolchain_info), 226 ) 227 228_BAZEL_TOOLS_PREFIX = "external/bazel_tools/" 229 230def _experimental_filter_sources(proto_info, proto_lang_toolchain_info): 231 if not proto_info.direct_sources: 232 return [], [] 233 234 # Collect a set of provided protos 235 provided_proto_sources = proto_lang_toolchain_info.provided_proto_sources 236 provided_paths = {} 237 for src in provided_proto_sources: 238 path = src.path 239 240 # For listed protos bundled with the Bazel tools repository, their exec paths start 241 # with external/bazel_tools/. This prefix needs to be removed first, because the protos in 242 # user repositories will not have that prefix. 243 if path.startswith(_BAZEL_TOOLS_PREFIX): 244 provided_paths[path[len(_BAZEL_TOOLS_PREFIX):]] = None 245 else: 246 provided_paths[path] = None 247 248 # Filter proto files 249 proto_files = proto_info._direct_proto_sources 250 excluded = [] 251 included = [] 252 for proto_file in proto_files: 253 if proto_file.path in provided_paths: 254 excluded.append(proto_file) 255 else: 256 included.append(proto_file) 257 return included, excluded 258 259def _experimental_should_generate_code( 260 proto_info, 261 proto_lang_toolchain_info, 262 rule_name, 263 target_label): 264 """Checks if the code should be generated for the given proto_library. 265 266 The code shouldn't be generated only when the toolchain already provides it 267 to the language through its runtime dependency. 268 269 It fails when the proto_library contains mixed proto files, that should and 270 shouldn't generate code. 271 272 Args: 273 proto_info: (ProtoInfo) The ProtoInfo from proto_library to check the generation for. 274 proto_lang_toolchain_info: (ProtoLangToolchainInfo) The proto lang toolchain info. 275 Obtained from a `proto_lang_toolchain` target or constructed ad-hoc. 276 rule_name: (str) Name of the rule used in the failure message. 277 target_label: (Label) The label of the target used in the failure message. 278 279 Returns: 280 (bool) True when the code should be generated. 281 """ 282 included, excluded = _experimental_filter_sources(proto_info, proto_lang_toolchain_info) 283 284 if included and excluded: 285 fail(("The 'srcs' attribute of '%s' contains protos for which '%s' " + 286 "shouldn't generate code (%s), in addition to protos for which it should (%s).\n" + 287 "Separate '%s' into 2 proto_library rules.") % ( 288 target_label, 289 rule_name, 290 ", ".join([f.short_path for f in excluded]), 291 ", ".join([f.short_path for f in included]), 292 target_label, 293 )) 294 295 return bool(included) 296 297def _declare_generated_files( 298 actions, 299 proto_info, 300 extension, 301 name_mapper = None): 302 """Declares generated files with a specific extension. 303 304 Use this in lang_proto_library-es when protocol compiler generates files 305 that correspond to .proto file names. 306 307 The function removes ".proto" extension with given one (e.g. ".pb.cc") and 308 declares new output files. 309 310 Args: 311 actions: (ActionFactory) Obtained by ctx.actions, used to declare the files. 312 proto_info: (ProtoInfo) The ProtoInfo to declare the files for. 313 extension: (str) The extension to use for generated files. 314 name_mapper: (str->str) A function mapped over the base filename without 315 the extension. Used it to replace characters in the name that 316 cause problems in a specific programming language. 317 318 Returns: 319 (list[File]) The list of declared files. 320 """ 321 proto_sources = proto_info.direct_sources 322 outputs = [] 323 324 for src in proto_sources: 325 basename_no_ext = src.basename[:-(len(src.extension) + 1)] 326 327 if name_mapper: 328 basename_no_ext = name_mapper(basename_no_ext) 329 330 # Note that two proto_library rules can have the same source file, so this is actually a 331 # shared action. NB: This can probably result in action conflicts if the proto_library rules 332 # are not the same. 333 outputs.append(actions.declare_file(basename_no_ext + extension, sibling = src)) 334 335 return outputs 336 337def _toolchain_type(proto_lang_toolchain_info): 338 if toolchains.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION: 339 return getattr(proto_lang_toolchain_info, "toolchain_type", None) 340 else: 341 return None 342 343proto_common = struct( 344 compile = _compile, 345 declare_generated_files = _declare_generated_files, 346 check_collocated = _check_collocated, 347 experimental_should_generate_code = _experimental_should_generate_code, 348 experimental_filter_sources = _experimental_filter_sources, 349 get_import_path = _get_import_path, 350 ProtoLangToolchainInfo = ProtoLangToolchainInfo, 351 INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION = toolchains.INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION, 352 INCOMPATIBLE_PASS_TOOLCHAIN_TYPE = ( 353 getattr(native_proto_common, "INCOMPATIBLE_PASS_TOOLCHAIN_TYPE", False) or 354 not hasattr(native_proto_common, "ProtoLangToolchainInfo") 355 ), 356) 357