1#!/usr/bin/env python3 2 3# Copyright 2016 gRPC authors. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18 19import errno 20import filecmp 21import glob 22import os 23import os.path 24import pprint 25import shutil 26import subprocess 27import sys 28import traceback 29import uuid 30 31# the template for the content of protoc_lib_deps.py 32DEPS_FILE_CONTENT = """ 33# Copyright 2017 gRPC authors. 34# 35# Licensed under the Apache License, Version 2.0 (the "License"); 36# you may not use this file except in compliance with the License. 37# You may obtain a copy of the License at 38# 39# http://www.apache.org/licenses/LICENSE-2.0 40# 41# Unless required by applicable law or agreed to in writing, software 42# distributed under the License is distributed on an "AS IS" BASIS, 43# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 44# See the License for the specific language governing permissions and 45# limitations under the License. 46 47# AUTO-GENERATED BY make_grpcio_tools.py! 48CC_FILES={cc_files} 49 50PROTO_FILES={proto_files} 51 52CC_INCLUDES={cc_includes} 53PROTO_INCLUDE={proto_include} 54 55{commit_hash_expr} 56""" 57 58# expose commit hash suffix and prefix for check_grpcio_tools.py 59COMMIT_HASH_PREFIX = 'PROTOBUF_SUBMODULE_VERSION="' 60COMMIT_HASH_SUFFIX = '"' 61 62EXTERNAL_LINKS = [ 63 ('@com_google_absl//', 'third_party/abseil-cpp/'), 64 ('@com_google_protobuf//', 'third_party/protobuf/'), 65 ('@utf8_range//:', 'third_party/utf8_range/'), 66] 67 68PROTOBUF_PROTO_PREFIX = '@com_google_protobuf//src/' 69 70# will be added to include path when building grpcio_tools 71CC_INCLUDES = [ 72 os.path.join('third_party', 'abseil-cpp'), 73 os.path.join('third_party', 'protobuf', 'src'), 74 os.path.join('third_party', 'utf8_range'), 75] 76 77# include path for .proto files 78PROTO_INCLUDE = os.path.join('third_party', 'protobuf', 'src') 79 80# the target directory is relative to the grpcio_tools package root. 81GRPCIO_TOOLS_ROOT_PREFIX = 'tools/distrib/python/grpcio_tools/' 82 83# Pairs of (source, target) directories to copy 84# from the grpc repo root to the grpcio_tools build root. 85COPY_FILES_SOURCE_TARGET_PAIRS = [ 86 ('include', 'grpc_root/include'), 87 ('src/compiler', 'grpc_root/src/compiler'), 88 ('third_party/abseil-cpp/absl', 'third_party/abseil-cpp/absl'), 89 ('third_party/protobuf/src', 'third_party/protobuf/src'), 90 ('third_party/utf8_range', 'third_party/utf8_range') 91] 92 93# grpc repo root 94GRPC_ROOT = os.path.abspath( 95 os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')) 96 97# the directory under which to probe for the current protobuf commit SHA 98GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 99 'protobuf') 100 101# the file to generate 102GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 103 'python', 'grpcio_tools', 104 'protoc_lib_deps.py') 105 106# the script to run for getting dependencies 107BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 108 'bazel_deps.sh') 109 110# the bazel target to scrape to get list of sources for the build 111BAZEL_DEPS_PROTOC_LIB_QUERY = '@com_google_protobuf//:protoc_lib' 112 113BAZEL_DEPS_COMMON_PROTOS_QUERIES = [ 114 '@com_google_protobuf//:well_known_type_protos', 115 # has both plugin.proto and descriptor.proto 116 '@com_google_protobuf//:compiler_plugin_proto', 117] 118 119 120def protobuf_submodule_commit_hash(): 121 """Gets the commit hash for the HEAD of the protobuf submodule currently 122 checked out.""" 123 cwd = os.getcwd() 124 os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT) 125 output = subprocess.check_output(['git', 'rev-parse', 'HEAD']) 126 os.chdir(cwd) 127 return output.decode("ascii").splitlines()[0].strip() 128 129 130def _bazel_query(query): 131 """Runs 'bazel query' to collect source file info.""" 132 print('Running "bazel query %s"' % query) 133 output = subprocess.check_output([BAZEL_DEPS, query]) 134 return output.decode("ascii").splitlines() 135 136 137def _pretty_print_list(items): 138 """Pretty print python list""" 139 formatted = pprint.pformat(items, indent=4) 140 # add newline after opening bracket (and fix indent of the next line) 141 if formatted.startswith('['): 142 formatted = formatted[0] + '\n ' + formatted[1:] 143 # add newline before closing bracket 144 if formatted.endswith(']'): 145 formatted = formatted[:-1] + '\n' + formatted[-1] 146 return formatted 147 148 149def _bazel_name_to_file_path(name): 150 """Transform bazel reference to source file name.""" 151 for link in EXTERNAL_LINKS: 152 if name.startswith(link[0]): 153 filepath = link[1] + name[len(link[0]):].replace(':', '/') 154 155 # For some reason, the WKT sources (such as wrappers.pb.cc) 156 # end up being reported by bazel as having an extra 'wkt/google/protobuf' 157 # in path. Removing it makes the compilation pass. 158 # TODO(jtattermusch) Get dir of this hack. 159 return filepath.replace('wkt/google/protobuf/', '') 160 return None 161 162 163def _generate_deps_file_content(): 164 """Returns the data structure with dependencies of protoc as python code.""" 165 cc_files_output = _bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY) 166 167 # Collect .cc files (that will be later included in the native extension build) 168 cc_files = [] 169 for name in cc_files_output: 170 if name.endswith('.cc'): 171 filepath = _bazel_name_to_file_path(name) 172 if filepath: 173 cc_files.append(filepath) 174 175 # Collect list of .proto files that will be bundled in the grpcio_tools package. 176 raw_proto_files = [] 177 for target in BAZEL_DEPS_COMMON_PROTOS_QUERIES: 178 raw_proto_files += _bazel_query(target) 179 proto_files = [ 180 name[len(PROTOBUF_PROTO_PREFIX):].replace(':', '/') 181 for name in raw_proto_files 182 if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX) 183 ] 184 185 commit_hash = protobuf_submodule_commit_hash() 186 commit_hash_expr = COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX 187 188 deps_file_content = DEPS_FILE_CONTENT.format( 189 cc_files=_pretty_print_list(sorted(cc_files)), 190 proto_files=_pretty_print_list(sorted(set(proto_files))), 191 cc_includes=_pretty_print_list(CC_INCLUDES), 192 proto_include=repr(PROTO_INCLUDE), 193 commit_hash_expr=commit_hash_expr) 194 return deps_file_content 195 196 197def _copy_source_tree(source, target): 198 """Copies source directory to a given target directory.""" 199 print('Copying contents of %s to %s' % (source, target)) 200 # TODO(jtattermusch): It is unclear why this legacy code needs to copy 201 # the source directory to the target via the following boilerplate. 202 # Should this code be simplified? 203 for source_dir, _, files in os.walk(source): 204 target_dir = os.path.abspath( 205 os.path.join(target, os.path.relpath(source_dir, source))) 206 try: 207 os.makedirs(target_dir) 208 except OSError as error: 209 if error.errno != errno.EEXIST: 210 raise 211 for relative_file in files: 212 source_file = os.path.abspath( 213 os.path.join(source_dir, relative_file)) 214 target_file = os.path.abspath( 215 os.path.join(target_dir, relative_file)) 216 shutil.copyfile(source_file, target_file) 217 218 219def main(): 220 os.chdir(GRPC_ROOT) 221 222 # Step 1: 223 # In order to be able to build the grpcio_tools package, we need the source code for the codegen plugins 224 # and its dependencies to be available under the build root of the grpcio_tools package. 225 # So we simply copy all the necessary files where the build will expect them to be. 226 for source, target in COPY_FILES_SOURCE_TARGET_PAIRS: 227 # convert the slashes in the relative path to platform-specific path dividers. 228 # All paths are relative to GRPC_ROOT 229 source_abs = os.path.join(GRPC_ROOT, os.path.join(*source.split('/'))) 230 # for targets, add grpcio_tools root prefix 231 target = GRPCIO_TOOLS_ROOT_PREFIX + target 232 target_abs = os.path.join(GRPC_ROOT, os.path.join(*target.split('/'))) 233 234 _copy_source_tree(source_abs, target_abs) 235 print( 236 'The necessary source files were copied under the grpcio_tools package root.' 237 ) 238 print() 239 240 # Step 2: 241 # Extract build metadata from bazel build (by running "bazel query") 242 # and populate the protoc_lib_deps.py file with python-readable data structure 243 # that will be used by grpcio_tools's setup.py (so it knows how to configure 244 # the native build for the codegen plugin) 245 try: 246 print('Invoking "bazel query" to gather the protobuf dependencies.') 247 protoc_lib_deps_content = _generate_deps_file_content() 248 except Exception as error: 249 # We allow this script to succeed even if we couldn't get the dependencies, 250 # as then we can assume that even without a successful bazel run the 251 # dependencies currently in source control are 'good enough'. 252 sys.stderr.write("Got non-fatal error:\n") 253 traceback.print_exc(file=sys.stderr) 254 return 255 # If we successfully got the dependencies, truncate and rewrite the deps file. 256 with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: 257 deps_file.write(protoc_lib_deps_content) 258 print('File "%s" updated.' % GRPC_PYTHON_PROTOC_LIB_DEPS) 259 print('Done.') 260 261 262if __name__ == '__main__': 263 main() 264