• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Description:
2#  Tools for building the TensorFlow pip package.
3
4load("//tensorflow:tensorflow.bzl", "filegroup_as_file", "if_windows", "transitive_hdrs")
5load("//third_party/mkl:build_defs.bzl", "if_enable_mkl", "if_mkl", "if_mkl_ml")
6load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
7load("@local_config_syslibs//:build_defs.bzl", "if_not_system_lib")
8load("//tensorflow/core/platform:build_config_root.bzl", "tf_additional_license_deps")
9
10package(default_visibility = ["//visibility:private"])
11
12# This returns a list of headers of all public header libraries (e.g.,
13# framework, lib), and all of the transitive dependencies of those
14# public headers.  Not all of the headers returned by the filegroup
15# are public (e.g., internal headers that are included by public
16# headers), but the internal headers need to be packaged in the
17# pip_package for the public headers to be properly included.
18#
19# Public headers are therefore defined by those that are both:
20#
21# 1) "publicly visible" as defined by bazel
22# 2) Have documentation.
23#
24# This matches the policy of "public" for our python API.
25transitive_hdrs(
26    name = "included_headers",
27    deps = [
28        "//tensorflow/compiler/tf2xla:xla_compiled_cpu_function",
29        "//tensorflow/compiler/mlir:mlir_graph_optimization_pass",
30        "//tensorflow/core:core_cpu",
31        "//tensorflow/core:framework",
32        "//tensorflow/core:lib",
33        "//tensorflow/core:protos_all_cc",
34        "//tensorflow/core/platform:stream_executor",
35        "//tensorflow/cc/saved_model:loader",
36        "//tensorflow/cc/saved_model:reader",
37        "//tensorflow/cc/saved_model:bundle_v2",
38        "//tensorflow/c/experimental/filesystem:filesystem_interface",
39        "//tensorflow/c/experimental/pluggable_profiler:pluggable_profiler_hdrs",
40        "//tensorflow/c/experimental/stream_executor:stream_executor_hdrs",
41        "//tensorflow/c/experimental/grappler:grappler_hdrs",
42        "//tensorflow/c:kernels_hdrs",
43        "//tensorflow/c:ops_hdrs",
44        # TODO(rostam): Revisit these after cc_shared_library everywhere
45        "//tensorflow/cc:client_session",
46        "//tensorflow/cc:cc_ops",
47        # WARNING: None of the C/C++ code under python/ has any API guarantees, and TF team
48        # reserves the right to change APIs and other header-level interfaces.  If your custom
49        # op uses these headers, it may break when users upgrade their version of tensorflow.
50        # NOTE(ebrevdo): None of the python symbols are exported by libtensorflow_framework.so.
51        # Code that relies on these headers should dynamically link to
52        # _pywrap_tensorflow_internal.so as well.
53        "//tensorflow/python/grappler:model_analyzer_lib",
54        "//tensorflow/python/lib/core:py_exception_registry",
55        "//tensorflow/python/lib/core:pybind11_proto",
56        "//tensorflow/python/lib/core:pybind11_absl",
57        "//tensorflow/python/lib/core:pybind11_lib",
58        "//tensorflow/python/lib/core:pybind11_status",
59        "//tensorflow/python/lib/core:py_func_lib",
60        "//tensorflow/python/lib/core:py_seq_tensor",
61        "//tensorflow/python/lib/core:py_util",
62        "//tensorflow/python/util:cpp_python_util",
63        "//tensorflow/python/util:kernel_registry",
64        "//tensorflow/python/framework:python_op_gen",
65        "//tensorflow/python/client:tf_session_helper",
66        "//third_party/eigen3",
67    ] + if_cuda([
68        "@local_config_cuda//cuda:cuda_headers",
69    ]),
70)
71
72py_binary(
73    name = "simple_console",
74    srcs = ["simple_console.py"],
75    srcs_version = "PY3",
76    deps = ["//tensorflow:tensorflow_py"],
77)
78
79filegroup_as_file(
80    name = "xla_compiled_cpu_runtime_srcs.txt",
81    dep = "//tensorflow/compiler/tf2xla:xla_compiled_cpu_runtime_srcs",
82)
83
84filegroup(
85    name = "xla_cmake",
86    srcs = ["xla_build/CMakeLists.txt"],
87)
88
89# Add dynamic kernel dso files here.
90DYNAMIC_LOADED_KERNELS = [
91    "//tensorflow/core/kernels:libtfkernel_sobol_op.so",
92]
93
94COMMON_PIP_DEPS = [
95    ":licenses",
96    "MANIFEST.in",
97    "README",
98    "setup.py",
99    ":included_headers",
100    ":xla_compiled_cpu_runtime_srcs.txt_file",
101    ":xla_cmake",
102    "//tensorflow/compiler/tf2xla:xla_compiled_cpu_runtime_hdrs",
103    "//tensorflow/compiler/tf2xla:xla_compiled_cpu_runtime_srcs",
104    "//tensorflow/compiler/mlir/tensorflow:gen_mlir_passthrough_op_py",
105    "//tensorflow/core/function/trace_type:serialization_test_proto_py",
106    "//tensorflow/core/function/trace_type:serialization",
107    "//tensorflow/core:protos_all_proto_srcs",
108    "//tensorflow/lite/python:tflite_convert",
109    "//tensorflow/lite/toco/python:toco_from_protos",
110    "//tensorflow/lite/tools:visualize",
111    "//tensorflow/python/autograph/core:test_lib",
112    "//tensorflow/python/autograph/impl/testing:pybind_for_testing",
113    "//tensorflow/python/autograph/pyct/testing",
114    "//tensorflow/python/autograph/pyct/common_transformers:common_transformers",
115    "//tensorflow/python/compiler:compiler",
116    "//tensorflow/python:cond_v2",
117    "//tensorflow/python:distributed_framework_test_lib",
118    "//tensorflow/python/distribute:distribute_test_lib_pip",
119    "//tensorflow/python/training/experimental:loss_scale",
120    "//tensorflow/python/training/experimental:loss_scale_optimizer",
121    "//tensorflow/python/framework:meta_graph_testdata",
122    "//tensorflow/python/data/benchmarks:benchmark_base",
123    "//tensorflow/python/data/experimental/kernel_tests/service:multi_process_cluster",
124    "//tensorflow/python/data/experimental/kernel_tests/service:test_base",
125    "//tensorflow/python/data/experimental/ops:testing",
126    "//tensorflow/python/data/experimental/service:server_lib",
127    "//tensorflow/python/data/kernel_tests:checkpoint_test_base",
128    "//tensorflow/python/data/kernel_tests:test_base",
129    "//tensorflow/python/data/kernel_tests:tf_record_test_base",
130    "//tensorflow/python/ops/ragged:ragged_tensor_test_ops",
131    "//tensorflow/python/debug:debug_pip",
132    "//tensorflow/python/distribute:combinations",
133    "//tensorflow/python/distribute/failure_handling:check_preemption_py",
134    "//tensorflow/python/distribute/failure_handling:failure_handling_lib",
135    "//tensorflow/python/distribute/failure_handling:gce_util",
136    "//tensorflow/python/distribute:multi_process_runner",
137    "//tensorflow/python/eager:eager_pip",
138    "//tensorflow/python/keras:combinations",
139    "//tensorflow/python/keras/mixed_precision:test_util",
140    "//tensorflow/python/keras/utils:dataset_creator",
141    "//tensorflow/python/kernel_tests/linalg/testdata:self_adjoint_eig_op_test_files",
142    "//tensorflow/python/kernel_tests/nn_ops:bias_op_base",
143    "//tensorflow/python/kernel_tests/nn_ops:cudnn_deterministic_base",
144    "//tensorflow/python/kernel_tests/nn_ops:depthwise_conv_op_base",
145    "//tensorflow/python/kernel_tests/nn_ops:xent_op_test_base",
146    "//tensorflow/python/kernel_tests/random:util",
147    "//tensorflow/python/kernel_tests/signal:test_util",
148    "//tensorflow/python/kernel_tests/sparse_ops:sparse_xent_op_test_base",
149    "//tensorflow/python/ops/structured:structured_tensor_dynamic",
150    "//tensorflow/python/profiler:trace",
151    "//tensorflow/python/saved_model:saved_model",
152    "//tensorflow/python/tools:tools_pip",
153    "//tensorflow/python/tools/api/generator:create_python_api",
154    "//tensorflow/python/tpu",
155    "//tensorflow/python/util:example_parser_configuration",
156    "//tensorflow/python:image_grad_test_base",
157    "//tensorflow/python:memory_checker",
158    "//tensorflow/python:test_ops",
159    "//tensorflow/python:while_v2",
160    "//tensorflow/tools/common:public_api",
161    "//tensorflow/tools/common:test_module1",
162    "//tensorflow/tools/common:traverse",
163    "//tensorflow/python/distribute:parameter_server_strategy_v2",
164    "//tensorflow/python/distribute/coordinator:cluster_coordinator",
165    "//tensorflow/python/distribute/coordinator:remote_eager_lib",
166    "//tensorflow/python/distribute/coordinator:metric_utils",
167    "//tensorflow/python/distribute/experimental/rpc:rpc_ops",
168    "//tensorflow:tensorflow_py",
169    "//tensorflow/tools/docs:tf_doctest_lib",
170]
171
172# On Windows, python binary is a zip file of runfiles tree.
173# Add everything to its data dependency for generating a runfiles tree
174# for building the pip package on Windows.
175py_binary(
176    name = "simple_console_for_windows",
177    srcs = ["simple_console_for_windows.py"],
178    data = COMMON_PIP_DEPS + [
179        "//tensorflow/python:pywrap_tensorflow_import_lib_file",
180    ],
181    srcs_version = "PY3",
182    deps = ["//tensorflow:tensorflow_py"],
183)
184
185filegroup(
186    name = "licenses",
187    data = [
188        "//:LICENSE",
189        "//third_party/eigen3:LICENSE",
190        "//third_party/fft2d:LICENSE",
191        "//third_party/icu/data:LICENSE",
192        "@FP16//:LICENSE",
193        "@FXdiv//:LICENSE",
194        "@XNNPACK//:LICENSE",
195        "@arm_neon_2_x86_sse//:LICENSE",
196        "@astunparse_archive//:LICENSE",
197        "@boringssl//:LICENSE",
198        "@com_google_absl//:LICENSE",
199        "@com_google_protobuf//:LICENSE",
200        "@com_googlesource_code_re2//:LICENSE",
201        "@cpuinfo//:LICENSE",
202        "@cpuinfo//deps/clog:LICENSE",
203        "@curl//:COPYING",
204        "@dill_archive//:LICENSE",
205        "@dlpack//:LICENSE",
206        "@double_conversion//:LICENSE",
207        "@eigen_archive//:COPYING.MPL2",
208        "@farmhash_archive//:COPYING",
209        "@fft2d//:readme2d.txt",
210        "@flatbuffers//:LICENSE.txt",
211        "@gast_archive//:PKG-INFO",
212        "@gemmlowp//:LICENSE",
213        "@gif//:COPYING",
214        "@highwayhash//:LICENSE",
215        "@icu//:icu4c/LICENSE",
216        "@kissfft//:COPYING",
217        "@libjpeg_turbo//:LICENSE.md",
218        "@llvm-project//llvm:LICENSE.TXT",
219        "@llvm-project//mlir:LICENSE.TXT",
220        "@lmdb//:LICENSE",
221        "@local_config_tensorrt//:LICENSE",
222        "@nasm//:LICENSE",
223        "@nsync//:LICENSE",
224        "@opt_einsum_archive//:LICENSE",
225        "@pasta//:LICENSE",
226        "@png//:LICENSE",
227        "@pthreadpool//:LICENSE",
228        "@ruy//:LICENSE",
229        "@six_archive//:LICENSE",
230        "@snappy//:COPYING",
231        "@sobol_data//:LICENSE",
232        "@tblib_archive//:LICENSE",
233        "@termcolor_archive//:COPYING.txt",
234        "@typing_extensions_archive//:LICENSE",
235        "@zlib//:zlib.h",
236    ] + select({
237        "//tensorflow:android": [],
238        "//tensorflow:ios": [],
239        "//tensorflow:linux_s390x": [],
240        "//tensorflow:windows": [],
241        "//conditions:default": [],
242    }) + select({
243        "//tensorflow:android": [],
244        "//tensorflow:ios": [],
245        "//tensorflow:linux_s390x": [],
246        "//tensorflow:windows": [],
247        "//conditions:default": [
248            "@com_github_googlecloudplatform_google_cloud_cpp//:LICENSE",
249        ],
250    }) + select({
251        "//tensorflow/core/kernels:xsmm": [
252            "@libxsmm_archive//:LICENSE.md",
253        ],
254        "//conditions:default": [],
255    }) + select({
256        "//tensorflow:with_numa_support": [
257            "@hwloc//:COPYING",
258        ],
259        "//conditions:default": [],
260    }) + if_cuda([
261        "@cub_archive//:LICENSE.TXT",
262        "@local_config_nccl//:LICENSE",
263    ]) + if_mkl([
264        "//third_party/mkl_dnn:LICENSE",
265    ]) + if_enable_mkl(["//third_party/mkl:LICENSE"]) + if_not_system_lib(
266        "absl_py",
267        [
268            "@absl_py//:LICENSE",
269        ],
270    ) + if_not_system_lib(
271        "com_github_grpc_grpc",
272        [
273            "@com_github_grpc_grpc//:LICENSE",
274            "@com_github_grpc_grpc//third_party/address_sorting:LICENSE",
275        ],
276    ) + tf_additional_license_deps(),
277)
278
279sh_binary(
280    name = "build_pip_package",
281    srcs = ["build_pip_package.sh"],
282    data = COMMON_PIP_DEPS +
283           select({
284               "//tensorflow:windows": [
285                   ":simple_console_for_windows",
286               ],
287               "//conditions:default": [
288                   ":simple_console",
289               ],
290           }) +
291           select({
292               "//tensorflow:dynamic_loaded_kernels": DYNAMIC_LOADED_KERNELS,
293               "//conditions:default": [],
294           }) + if_mkl_ml(["//third_party/mkl:intel_binary_blob"]),
295    visibility = ["//visibility:public"],
296)
297
298# A genrule for generating a marker file for the pip package on Windows
299#
300# This only works on Windows, because :simple_console_for_windows is a
301# python zip file containing everything we need for building the pip package.
302# However, on other platforms, due to https://github.com/bazelbuild/bazel/issues/4223,
303# when C++ extensions change, this generule doesn't rebuild.
304genrule(
305    name = "win_pip_package_marker",
306    srcs = if_windows([
307        ":build_pip_package",
308        ":simple_console_for_windows",
309    ]),
310    outs = ["win_pip_package_marker_file"],
311    cmd = select({
312        "//conditions:default": "touch $@",
313        "//tensorflow:windows": "sha1sum $(locations :build_pip_package) $(locations :simple_console_for_windows) > $@",
314    }),
315    visibility = ["//visibility:public"],
316)
317