1 /* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #ifndef TENSORFLOW_CORE_TPU_KERNELS_TPU_COMPILATION_CACHE_SERVICE_H_ 16 #define TENSORFLOW_CORE_TPU_KERNELS_TPU_COMPILATION_CACHE_SERVICE_H_ 17 18 #include <atomic> 19 #include <memory> 20 21 #include "grpcpp/server_builder.h" 22 #include "tensorflow/core/distributed_runtime/rpc/grpc_call.h" 23 #include "tensorflow/core/lib/core/threadpool.h" 24 #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_common.pb.h" 25 #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_grpc.h" 26 #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" 27 28 namespace tensorflow { 29 // gRPC service for handling CompilationCache requests. 30 // To avoid OOMs during execution, this service using the asynchronous raw gRPC 31 // interface to serialize cache results directly to gRPC byte buffers. This 32 // allows us to control serialization concurrency and avoids making an extra 33 // copy of the program cache for each worker. 34 class TpuCompilationCacheService { 35 public: 36 using ServiceType = ::tensorflow::tpu::grpc::TpuCompilationCacheService; 37 using AsyncService = ServiceType::AsyncService; 38 39 TpuCompilationCacheService(::grpc::ServerBuilder* server_builder, 40 tpu::TpuCompilationCacheInterface* cache); 41 ~TpuCompilationCacheService(); 42 43 void Start(); 44 bool Shutdown(int timeout_sec); 45 void SetMemoryQuota(size_t max_bytes); 46 47 private: 48 void HandleRPCsLoop(); 49 50 using GetTpuProgramCall = Call<TpuCompilationCacheService, AsyncService, 51 tpu::GetTpuProgramRequest, ::grpc::ByteBuffer>; 52 53 // Schedule the cache fetch into the serving thread pool. 54 void HandleGetTpuProgram(GetTpuProgramCall* call); 55 56 // Performs the actual cache fetch and serialization. 57 void GetTpuProgram(GetTpuProgramCall* call); 58 59 std::atomic<bool> running_; 60 tpu::TpuCompilationCacheInterface* cache_; 61 ::grpc::ServerBuilder* server_builder_; 62 std::unique_ptr<::grpc::Server> server_; 63 std::unique_ptr<::grpc::ServerCompletionQueue> cq_; 64 std::unique_ptr<thread::ThreadPool> thread_pool_; 65 std::unique_ptr<Thread> polling_thread_; 66 AsyncService service_; 67 }; 68 } // namespace tensorflow 69 70 #endif // TENSORFLOW_CORE_TPU_KERNELS_TPU_COMPILATION_CACHE_SERVICE_H_ 71