1# Copyright 2024 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Internal implementation of proto compilation. 15 16# Overview of implementation 17 18(If you just want to use the macros, see their docstrings; this section is 19intended to orient future maintainers.) 20 21Proto code generation is carried out by the pwpb_proto_library, 22nanopb_proto_library, pw_raw_rpc_proto_library and pw_nanopb_rpc_proto_library 23rules using aspects 24(https://docs.bazel.build/versions/main/skylark/aspects.html). 25 26As an example, pwpb_proto_library has a single proto_library as a dependency, 27but that proto_library may depend on other proto_library targets; as a result, 28the generated .pwpb.h file #include's .pwpb.h files generated from the 29dependency proto_libraries. The aspect propagates along the proto_library 30dependency graph, running the proto compiler on each proto_library in the 31original target's transitive dependencies, ensuring that we're not missing any 32.pwpb.h files at C++ compile time. 33 34Although we have a separate rule for each protocol compiler plugin 35(pwpb_proto_library, nanopb_proto_library, pw_raw_rpc_proto_library, 36pw_nanopb_rpc_proto_library), they actually share an implementation 37(compile_proto) and use similar aspects, all generated by 38proto_compiler_aspect. 39""" 40 41load("@bazel_skylib//lib:paths.bzl", "paths") 42load("@com_google_protobuf//bazel/common:proto_info.bzl", "ProtoInfo") 43load( 44 "//pw_build/bazel_internal:pigweed_internal.bzl", 45 _compile_cc = "compile_cc", 46) 47load("//pw_protobuf_compiler:pw_proto_filegroup.bzl", "PwProtoOptionsInfo") 48 49PwProtoInfo = provider( 50 "Returned by PW proto compilation aspect", 51 fields = { 52 "hdrs": "generated C++ header files", 53 "includes": "include paths for generated C++ header files", 54 "srcs": "generated C++ src files", 55 }, 56) 57 58def compile_proto(ctx): 59 """Implementation of the proto codegen rule. 60 61 The work of actually generating the code is done by the aspect, so here we 62 compile and return a CcInfo to link against. 63 64 Args: 65 ctx: Rule context object (https://bazel.build/rules/lib/builtins/ctx). 66 67 Returns: 68 A CcInfo provider. 69 """ 70 71 # Note that we don't distinguish between the files generated from the 72 # target, and the files generated from its dependencies. We return all of 73 # them together, and in pw_proto_library expose all of them as hdrs. 74 # Pigweed's plugins happen to only generate .h files, so this works, but 75 # strictly speaking we should expose only the files generated from the 76 # target itself in hdrs, and place the headers generated from dependencies 77 # in srcs. We don't perform layering_check in Pigweed, so this is not a big 78 # deal. 79 # 80 # TODO: b/234873954 - Tidy this up. 81 all_srcs = [] 82 all_hdrs = [] 83 all_includes = [] 84 for dep in ctx.attr.protos: 85 for f in dep[PwProtoInfo].hdrs: 86 all_hdrs.append(f) 87 for f in dep[PwProtoInfo].srcs: 88 all_srcs.append(f) 89 for i in dep[PwProtoInfo].includes: 90 all_includes.append(i) 91 92 return _compile_cc( 93 ctx, 94 all_srcs, 95 all_hdrs, 96 ctx.attr.deps, 97 all_includes, 98 defines = [], 99 ) 100 101def _options_symlink_path(options_file, workspace_root, proto_source_root, import_prefix, strip_import_prefix): 102 path_in_module = paths.relativize(options_file.path, workspace_root) 103 104 if strip_import_prefix: 105 stripped_path = paths.relativize(path_in_module, strip_import_prefix.lstrip("/")) 106 else: 107 stripped_path = path_in_module 108 109 if import_prefix: 110 extended_path = paths.join(import_prefix, stripped_path) 111 else: 112 extended_path = stripped_path 113 114 return paths.join(proto_source_root, extended_path) 115 116def _proto_compiler_aspect_impl(target, ctx): 117 # List the files we will generate for this proto_library target. 118 proto_info = target[ProtoInfo] 119 120 srcs = [] 121 hdrs = [] 122 123 # Setup the output root for the plugin to point to targets output 124 # directory. This allows us to declare the location of the files that protoc 125 # will output in a way that `ctx.actions.declare_file` will understand, 126 # since it works relative to the target. 127 out_path = ctx.bin_dir.path 128 if target.label.workspace_root: 129 out_path += "/" + target.label.workspace_root 130 if target.label.package: 131 out_path += "/" + target.label.package 132 133 # Add location of headers to cc include path. 134 # Depending on prefix rules, the include path can be directly from the 135 # output path, or underneath the package. 136 includes = [out_path] 137 138 for src in proto_info.direct_sources: 139 # Get the relative import path for this .proto file. 140 src_rel = paths.relativize(src.path, proto_info.proto_source_root) 141 proto_dir = paths.dirname(src_rel) 142 143 # Add location of headers to cc include path. 144 includes.append("{}/{}".format(out_path, src.owner.package)) 145 146 for ext in ctx.attr._extensions: 147 # Declare all output files, in target package dir. 148 generated_filename = src.basename[:-len("proto")] + ext 149 if proto_dir: 150 out_file_name = "{}/{}".format( 151 proto_dir, 152 generated_filename, 153 ) 154 else: 155 out_file_name = generated_filename 156 157 out_file = ctx.actions.declare_file(out_file_name) 158 159 if ext.endswith(".h"): 160 hdrs.append(out_file) 161 else: 162 srcs.append(out_file) 163 164 # The proto_source_root may be prefixed with the output directory. But it 165 # may not. Ensure that there is no such prefix, which is intended to become 166 # the only case one day. See 167 # https://github.com/protocolbuffers/protobuf/blob/069a66850d1d8bb83c1ca1eb5bdee87525290584/bazel/private/proto_info.bzl#L154-L165 168 relative_proto_source_root = proto_info.proto_source_root 169 if relative_proto_source_root.startswith(out_path): 170 relative_proto_source_root = paths.relativize(relative_proto_source_root, out_path) 171 172 # Symlink the .options files into the proto_source_root, so that they can be 173 # found by protoc plugins regardless of [strip_]import_prefix attribute 174 # values. 175 # 176 # For example, say we have a proto_library in //a/b/BUILD.bazel with 177 # strip_import_prefix = b and import_prefix = xyz. Then, the `.proto` files 178 # will live in the directory, 179 # 180 # bazel-bin/a/b/_virtual_imports/a/xyz/ 181 # 182 # What we do here is move the `.options` files to the same directory. Later 183 # on, we'll provide `bazel-bin/a/b/_virtual_imports` to the protoc plugin's 184 # search path via `--custom_opt=-I`. This way, the proto and options files 185 # will be alongside each other, as the plugins expect. 186 symlinks = [] 187 for src in ctx.rule.attr.srcs: 188 if PwProtoOptionsInfo in src: 189 for options_file in src[PwProtoOptionsInfo].options_files.to_list(): 190 path_to_options_file = _options_symlink_path( 191 options_file, 192 target.label.workspace_root, 193 relative_proto_source_root, 194 ctx.rule.attr.import_prefix, 195 ctx.rule.attr.strip_import_prefix, 196 ) 197 options_symlink_out = ctx.actions.declare_file(path_to_options_file) 198 ctx.actions.symlink(output = options_symlink_out, target_file = options_file) 199 symlinks.append(options_symlink_out) 200 201 # List the `.options` files from any `pw_proto_filegroup` targets listed 202 # under this target's `srcs`. 203 options_files = [ 204 options_file 205 for src in ctx.rule.attr.srcs 206 if PwProtoOptionsInfo in src 207 for options_file in src[PwProtoOptionsInfo].options_files.to_list() 208 ] 209 210 args = ctx.actions.args() 211 for path in proto_info.transitive_proto_path.to_list(): 212 args.add("-I{}".format(path)) 213 214 args.add("--plugin=protoc-gen-custom={}".format(ctx.executable._protoc_plugin.path)) 215 args.add("--custom_opt=-I{}".format(paths.join(out_path, relative_proto_source_root))) 216 217 for plugin_option in ctx.attr._plugin_options: 218 # If the plugin supports directly specifying the location of the options files, pass them here. 219 if plugin_option == "--options-file={}": 220 for options_file in options_files: 221 plugin_options_arg = plugin_option.format(options_file.path) 222 args.add("--custom_opt={}".format(plugin_options_arg)) 223 continue 224 args.add("--custom_opt={}".format(plugin_option)) 225 226 args.add("--custom_out={}".format(out_path)) 227 args.add_all(proto_info.direct_sources) 228 229 all_tools = [ 230 ctx.executable._protoc, 231 ctx.executable._protoc_plugin, 232 ] 233 234 ctx.actions.run( 235 inputs = depset( 236 direct = proto_info.direct_sources + 237 proto_info.transitive_sources.to_list() + 238 options_files + symlinks, 239 transitive = [proto_info.transitive_descriptor_sets], 240 ), 241 progress_message = "Generating %s C++ files for %s" % (ctx.attr._extensions, ctx.label.name), 242 tools = all_tools, 243 outputs = srcs + hdrs, 244 executable = ctx.executable._protoc, 245 arguments = [args], 246 env = { 247 # This effectively pre-adopts 248 # https://github.com/nanopb/nanopb/pull/1038, silencing an annoying 249 # warning in nanopb 0.4.9.1. 250 "NANOPB_PB2_NO_REBUILD": "1", 251 # The nanopb protobuf plugin likes to compile some temporary protos 252 # next to source files. This forces them to be written to Bazel's 253 # genfiles directory. 254 "NANOPB_PB2_TEMP_DIR": str(ctx.genfiles_dir), 255 }, 256 ) 257 258 transitive_srcs = srcs 259 transitive_hdrs = hdrs 260 transitive_includes = includes 261 for dep in ctx.rule.attr.deps: 262 transitive_srcs += dep[PwProtoInfo].srcs 263 transitive_hdrs += dep[PwProtoInfo].hdrs 264 transitive_includes += dep[PwProtoInfo].includes 265 return [PwProtoInfo( 266 srcs = transitive_srcs, 267 hdrs = transitive_hdrs, 268 includes = transitive_includes, 269 )] 270 271def proto_compiler_aspect(extensions, protoc_plugin, plugin_options = []): 272 """Returns an aspect that runs the proto compiler. 273 274 The aspect propagates through the deps of proto_library targets, running 275 the proto compiler with the specified plugin for each of their source 276 files. The proto compiler is assumed to produce one output file per input 277 .proto file. That file is placed under bazel-bin at the same path as the 278 input file, but with the specified extension (i.e., with _extensions = [ 279 .pwpb.h], the aspect converts pw_log/log.proto into 280 bazel-bin/pw_log/log.pwpb.h). 281 282 The aspect returns a provider exposing all the File objects generated from 283 the dependency graph. 284 """ 285 return aspect( 286 attr_aspects = ["deps"], 287 attrs = { 288 "_extensions": attr.string_list(default = extensions), 289 "_plugin_options": attr.string_list( 290 default = plugin_options, 291 ), 292 "_protoc": attr.label( 293 default = Label("@com_google_protobuf//:protoc"), 294 executable = True, 295 cfg = "exec", 296 ), 297 "_protoc_plugin": attr.label( 298 default = Label(protoc_plugin), 299 executable = True, 300 cfg = "exec", 301 ), 302 }, 303 implementation = _proto_compiler_aspect_impl, 304 provides = [PwProtoInfo], 305 ) 306