• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2
3# Copyright 2023 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import errno
18import os
19import os.path
20import pprint
21import shutil
22import subprocess
23import sys
24import traceback
25
26# the template for the content of observability_lib_deps.py
27DEPS_FILE_CONTENT = """
28# Copyright 2023 gRPC authors.
29#
30# Licensed under the Apache License, Version 2.0 (the "License");
31# you may not use this file except in compliance with the License.
32# You may obtain a copy of the License at
33#
34#     http://www.apache.org/licenses/LICENSE-2.0
35#
36# Unless required by applicable law or agreed to in writing, software
37# distributed under the License is distributed on an "AS IS" BASIS,
38# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
39# See the License for the specific language governing permissions and
40# limitations under the License.
41
42# AUTO-GENERATED BY make_grpcio_observability.py!
43CC_FILES={cc_files}
44
45CC_INCLUDES={cc_includes}
46"""
47
48# maps bazel reference to actual path
49BAZEL_REFERENCE_LINK = [
50    ("@com_google_absl//", "third_party/abseil-cpp/"),
51    ("//src", "grpc_root/src"),
52]
53
54ABSL_INCLUDE = (os.path.join("third_party", "abseil-cpp"),)
55
56# will be added to include path when building grpcio_observability
57EXTENSION_INCLUDE_DIRECTORIES = ABSL_INCLUDE
58
59CC_INCLUDES = list(EXTENSION_INCLUDE_DIRECTORIES)
60
61# the target directory is relative to the grpcio_observability package root.
62GRPCIO_OBSERVABILITY_ROOT_PREFIX = "src/python/grpcio_observability/"
63
64# Pairs of (source, target) directories to copy
65# from the grpc repo root to the grpcio_observability build root.
66COPY_FILES_SOURCE_TARGET_PAIRS = [
67    ("include", "grpc_root/include"),
68    ("third_party/abseil-cpp/absl", "third_party/abseil-cpp/absl"),
69    ("src/core", "grpc_root/src/core"),
70]
71
72# grpc repo root
73GRPC_ROOT = os.path.abspath(
74    os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..")
75)
76
77
78# the file to generate
79GRPC_PYTHON_OBSERVABILITY_LIB_DEPS = os.path.join(
80    GRPC_ROOT,
81    "src",
82    "python",
83    "grpcio_observability",
84    "observability_lib_deps.py",
85)
86
87# the script to run for getting dependencies
88BAZEL_DEPS = os.path.join(
89    GRPC_ROOT, "tools", "distrib", "python", "bazel_deps.sh"
90)
91
92# the bazel target to scrape to get list of sources for the build
93BAZEL_DEPS_QUERIES = [
94    "//src/core:experiments",
95    "//src/core:slice",
96    "//src/core:ref_counted_string",
97]
98
99
100def _bazel_query(query):
101    """Runs 'bazel query' to collect source file info."""
102    print('Running "bazel query %s"' % query)
103    output = subprocess.check_output([BAZEL_DEPS, query])
104    return output.decode("ascii").splitlines()
105
106
107def _pretty_print_list(items):
108    """Pretty print python list"""
109    formatted = pprint.pformat(items, indent=4)
110    # add newline after opening bracket (and fix indent of the next line)
111    if formatted.startswith("["):
112        formatted = formatted[0] + "\n " + formatted[1:]
113    # add newline before closing bracket
114    if formatted.endswith("]"):
115        formatted = formatted[:-1] + "\n" + formatted[-1]
116    return formatted
117
118
119def _bazel_name_to_file_path(name):
120    """Transform bazel reference to source file name."""
121    for link in BAZEL_REFERENCE_LINK:
122        if name.startswith(link[0]):
123            filepath = link[1] + name[len(link[0]) :].replace(":", "/")
124            return filepath
125    return None
126
127
128def _generate_deps_file_content():
129    """Returns the data structure with dependencies of protoc as python code."""
130    cc_files_output = []
131    for query in BAZEL_DEPS_QUERIES:
132        cc_files_output += _bazel_query(query)
133
134    # Collect .cc files (that will be later included in the native extension build)
135    cc_files = set()
136    for name in cc_files_output:
137        if name.endswith(".cc"):
138            filepath = _bazel_name_to_file_path(name)
139            if filepath:
140                cc_files.add(filepath)
141
142    deps_file_content = DEPS_FILE_CONTENT.format(
143        cc_files=_pretty_print_list(sorted(list(cc_files))),
144        cc_includes=_pretty_print_list(CC_INCLUDES),
145    )
146    return deps_file_content
147
148
149def _copy_source_tree(source, target):
150    """Copies source directory to a given target directory."""
151    print("Copying contents of %s to %s" % (source, target))
152    for source_dir, _, files in os.walk(source):
153        target_dir = os.path.abspath(
154            os.path.join(target, os.path.relpath(source_dir, source))
155        )
156        try:
157            os.makedirs(target_dir)
158        except OSError as error:
159            if error.errno != errno.EEXIST:
160                raise
161        for relative_file in files:
162            source_file = os.path.abspath(
163                os.path.join(source_dir, relative_file)
164            )
165            target_file = os.path.abspath(
166                os.path.join(target_dir, relative_file)
167            )
168            shutil.copyfile(source_file, target_file)
169
170
171def main():
172    os.chdir(GRPC_ROOT)
173
174    # Step 1:
175    # In order to be able to build the grpcio_observability package, we need the source
176    # code for the plugins and its dependencies to be available under the build root of
177    # the grpcio_observability package.
178    # So we simply copy all the necessary files where the build will expect them to be.
179    for source, target in COPY_FILES_SOURCE_TARGET_PAIRS:
180        # convert the slashes in the relative path to platform-specific path dividers.
181        # All paths are relative to GRPC_ROOT
182        source_abs = os.path.join(GRPC_ROOT, os.path.join(*source.split("/")))
183        # for targets, add grpcio_observability root prefix
184        target = GRPCIO_OBSERVABILITY_ROOT_PREFIX + target
185        target_abs = os.path.join(GRPC_ROOT, os.path.join(*target.split("/")))
186        _copy_source_tree(source_abs, target_abs)
187    print(
188        "The necessary source files were copied under the grpcio_observability package root."
189    )
190
191    # Step 2:
192    # Extract build metadata from bazel build (by running "bazel query")
193    # and populate the observability_lib_deps.py file with python-readable data structure
194    # that will be used by grpcio_observability's setup.py.
195    try:
196        print('Invoking "bazel query" to gather the dependencies.')
197        observability_lib_deps_content = _generate_deps_file_content()
198    except Exception as error:
199        # We allow this script to succeed even if we couldn't get the dependencies,
200        # as then we can assume that even without a successful bazel run the
201        # dependencies currently in source control are 'good enough'.
202        sys.stderr.write("Got non-fatal error:\n")
203        traceback.print_exc(file=sys.stderr)
204        return
205    # If we successfully got the dependencies, truncate and rewrite the deps file.
206    with open(GRPC_PYTHON_OBSERVABILITY_LIB_DEPS, "w") as deps_file:
207        deps_file.write(observability_lib_deps_content)
208    print('File "%s" updated.' % GRPC_PYTHON_OBSERVABILITY_LIB_DEPS)
209    print("Done.")
210
211
212if __name__ == "__main__":
213    main()
214