1 /* 2 * 3 * Copyright 2015-2016 gRPC authors. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 * 17 */ 18 19 /// A completion queue implements a concurrent producer-consumer queue, with 20 /// two main API-exposed methods: \a Next and \a AsyncNext. These 21 /// methods are the essential component of the gRPC C++ asynchronous API. 22 /// There is also a \a Shutdown method to indicate that a given completion queue 23 /// will no longer have regular events. This must be called before the 24 /// completion queue is destroyed. 25 /// All completion queue APIs are thread-safe and may be used concurrently with 26 /// any other completion queue API invocation; it is acceptable to have 27 /// multiple threads calling \a Next or \a AsyncNext on the same or different 28 /// completion queues, or to call these methods concurrently with a \a Shutdown 29 /// elsewhere. 30 /// \remark{All other API calls on completion queue should be completed before 31 /// a completion queue destructor is called.} 32 #ifndef GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_H 33 #define GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_H 34 35 #include <grpc/impl/codegen/atm.h> 36 #include <grpcpp/impl/codegen/completion_queue_tag.h> 37 #include <grpcpp/impl/codegen/core_codegen_interface.h> 38 #include <grpcpp/impl/codegen/grpc_library.h> 39 #include <grpcpp/impl/codegen/status.h> 40 #include <grpcpp/impl/codegen/time.h> 41 42 struct grpc_completion_queue; 43 44 namespace grpc { 45 46 template <class R> 47 class ClientReader; 48 template <class W> 49 class ClientWriter; 50 template <class W, class R> 51 class ClientReaderWriter; 52 template <class R> 53 class ServerReader; 54 template <class W> 55 class ServerWriter; 56 namespace internal { 57 template <class W, class R> 58 class ServerReaderWriterBody; 59 } // namespace internal 60 61 class Channel; 62 class ChannelInterface; 63 class ClientContext; 64 class CompletionQueue; 65 class Server; 66 class ServerBuilder; 67 class ServerContext; 68 class ServerInterface; 69 70 namespace internal { 71 class CompletionQueueTag; 72 class RpcMethod; 73 template <class ServiceType, class RequestType, class ResponseType> 74 class RpcMethodHandler; 75 template <class ServiceType, class RequestType, class ResponseType> 76 class ClientStreamingHandler; 77 template <class ServiceType, class RequestType, class ResponseType> 78 class ServerStreamingHandler; 79 template <class ServiceType, class RequestType, class ResponseType> 80 class BidiStreamingHandler; 81 template <class Streamer, bool WriteNeeded> 82 class TemplatedBidiStreamingHandler; 83 template <StatusCode code> 84 class ErrorMethodHandler; 85 template <class InputMessage, class OutputMessage> 86 class BlockingUnaryCallImpl; 87 } // namespace internal 88 89 extern CoreCodegenInterface* g_core_codegen_interface; 90 91 /// A thin wrapper around \ref grpc_completion_queue (see \ref 92 /// src/core/lib/surface/completion_queue.h). 93 /// See \ref doc/cpp/perf_notes.md for notes on best practices for high 94 /// performance servers. 95 class CompletionQueue : private GrpcLibraryCodegen { 96 public: 97 /// Default constructor. Implicitly creates a \a grpc_completion_queue 98 /// instance. CompletionQueue()99 CompletionQueue() 100 : CompletionQueue(grpc_completion_queue_attributes{ 101 GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING, 102 nullptr}) {} 103 104 /// Wrap \a take, taking ownership of the instance. 105 /// 106 /// \param take The completion queue instance to wrap. Ownership is taken. 107 explicit CompletionQueue(grpc_completion_queue* take); 108 109 /// Destructor. Destroys the owned wrapped completion queue / instance. ~CompletionQueue()110 ~CompletionQueue() { 111 g_core_codegen_interface->grpc_completion_queue_destroy(cq_); 112 } 113 114 /// Tri-state return for AsyncNext: SHUTDOWN, GOT_EVENT, TIMEOUT. 115 enum NextStatus { 116 SHUTDOWN, ///< The completion queue has been shutdown and fully-drained 117 GOT_EVENT, ///< Got a new event; \a tag will be filled in with its 118 ///< associated value; \a ok indicating its success. 119 TIMEOUT ///< deadline was reached. 120 }; 121 122 /// Read from the queue, blocking until an event is available or the queue is 123 /// shutting down. 124 /// 125 /// \param tag[out] Updated to point to the read event's tag. 126 /// \param ok[out] true if read a successful event, false otherwise. 127 /// 128 /// Note that each tag sent to the completion queue (through RPC operations 129 /// or alarms) will be delivered out of the completion queue by a call to 130 /// Next (or a related method), regardless of whether the operation succeeded 131 /// or not. Success here means that this operation completed in the normal 132 /// valid manner. 133 /// 134 /// Server-side RPC request: \a ok indicates that the RPC has indeed 135 /// been started. If it is false, the server has been Shutdown 136 /// before this particular call got matched to an incoming RPC. 137 /// 138 /// Client-side StartCall/RPC invocation: \a ok indicates that the RPC is 139 /// going to go to the wire. If it is false, it not going to the wire. This 140 /// would happen if the channel is either permanently broken or 141 /// transiently broken but with the fail-fast option. (Note that async unary 142 /// RPCs don't post a CQ tag at this point, nor do client-streaming 143 /// or bidi-streaming RPCs that have the initial metadata corked option set.) 144 /// 145 /// Client-side Write, Client-side WritesDone, Server-side Write, 146 /// Server-side Finish, Server-side SendInitialMetadata (which is 147 /// typically included in Write or Finish when not done explicitly): 148 /// \a ok means that the data/metadata/status/etc is going to go to the 149 /// wire. If it is false, it not going to the wire because the call 150 /// is already dead (i.e., canceled, deadline expired, other side 151 /// dropped the channel, etc). 152 /// 153 /// Client-side Read, Server-side Read, Client-side 154 /// RecvInitialMetadata (which is typically included in Read if not 155 /// done explicitly): \a ok indicates whether there is a valid message 156 /// that got read. If not, you know that there are certainly no more 157 /// messages that can ever be read from this stream. For the client-side 158 /// operations, this only happens because the call is dead. For the 159 /// server-sider operation, though, this could happen because the client 160 /// has done a WritesDone already. 161 /// 162 /// Client-side Finish: \a ok should always be true 163 /// 164 /// Server-side AsyncNotifyWhenDone: \a ok should always be true 165 /// 166 /// Alarm: \a ok is true if it expired, false if it was canceled 167 /// 168 /// \return true if got an event, false if the queue is fully drained and 169 /// shut down. Next(void ** tag,bool * ok)170 bool Next(void** tag, bool* ok) { 171 return (AsyncNextInternal(tag, ok, 172 g_core_codegen_interface->gpr_inf_future( 173 GPR_CLOCK_REALTIME)) != SHUTDOWN); 174 } 175 176 /// Read from the queue, blocking up to \a deadline (or the queue's shutdown). 177 /// Both \a tag and \a ok are updated upon success (if an event is available 178 /// within the \a deadline). A \a tag points to an arbitrary location usually 179 /// employed to uniquely identify an event. 180 /// 181 /// \param tag[out] Upon sucess, updated to point to the event's tag. 182 /// \param ok[out] Upon sucess, true if a successful event, false otherwise 183 /// See documentation for CompletionQueue::Next for explanation of ok 184 /// \param deadline[in] How long to block in wait for an event. 185 /// 186 /// \return The type of event read. 187 template <typename T> AsyncNext(void ** tag,bool * ok,const T & deadline)188 NextStatus AsyncNext(void** tag, bool* ok, const T& deadline) { 189 TimePoint<T> deadline_tp(deadline); 190 return AsyncNextInternal(tag, ok, deadline_tp.raw_time()); 191 } 192 193 /// EXPERIMENTAL 194 /// First executes \a F, then reads from the queue, blocking up to 195 /// \a deadline (or the queue's shutdown). 196 /// Both \a tag and \a ok are updated upon success (if an event is available 197 /// within the \a deadline). A \a tag points to an arbitrary location usually 198 /// employed to uniquely identify an event. 199 /// 200 /// \param F[in] Function to execute before calling AsyncNext on this queue. 201 /// \param tag[out] Upon sucess, updated to point to the event's tag. 202 /// \param ok[out] Upon sucess, true if read a regular event, false otherwise. 203 /// \param deadline[in] How long to block in wait for an event. 204 /// 205 /// \return The type of event read. 206 template <typename T, typename F> DoThenAsyncNext(F && f,void ** tag,bool * ok,const T & deadline)207 NextStatus DoThenAsyncNext(F&& f, void** tag, bool* ok, const T& deadline) { 208 CompletionQueueTLSCache cache = CompletionQueueTLSCache(this); 209 f(); 210 if (cache.Flush(tag, ok)) { 211 return GOT_EVENT; 212 } else { 213 return AsyncNext(tag, ok, deadline); 214 } 215 } 216 217 /// Request the shutdown of the queue. 218 /// 219 /// \warning This method must be called at some point if this completion queue 220 /// is accessed with Next or AsyncNext. \a Next will not return false 221 /// until this method has been called and all pending tags have been drained. 222 /// (Likewise for \a AsyncNext returning \a NextStatus::SHUTDOWN .) 223 /// Only once either one of these methods does that (that is, once the queue 224 /// has been \em drained) can an instance of this class be destroyed. 225 /// Also note that applications must ensure that no work is enqueued on this 226 /// completion queue after this method is called. 227 void Shutdown(); 228 229 /// Returns a \em raw pointer to the underlying \a grpc_completion_queue 230 /// instance. 231 /// 232 /// \warning Remember that the returned instance is owned. No transfer of 233 /// owership is performed. cq()234 grpc_completion_queue* cq() { return cq_; } 235 236 protected: 237 /// Private constructor of CompletionQueue only visible to friend classes CompletionQueue(const grpc_completion_queue_attributes & attributes)238 CompletionQueue(const grpc_completion_queue_attributes& attributes) { 239 cq_ = g_core_codegen_interface->grpc_completion_queue_create( 240 g_core_codegen_interface->grpc_completion_queue_factory_lookup( 241 &attributes), 242 &attributes, NULL); 243 InitialAvalanching(); // reserve this for the future shutdown 244 } 245 246 private: 247 // Friend synchronous wrappers so that they can access Pluck(), which is 248 // a semi-private API geared towards the synchronous implementation. 249 template <class R> 250 friend class ::grpc::ClientReader; 251 template <class W> 252 friend class ::grpc::ClientWriter; 253 template <class W, class R> 254 friend class ::grpc::ClientReaderWriter; 255 template <class R> 256 friend class ::grpc::ServerReader; 257 template <class W> 258 friend class ::grpc::ServerWriter; 259 template <class W, class R> 260 friend class ::grpc::internal::ServerReaderWriterBody; 261 template <class ServiceType, class RequestType, class ResponseType> 262 friend class ::grpc::internal::RpcMethodHandler; 263 template <class ServiceType, class RequestType, class ResponseType> 264 friend class ::grpc::internal::ClientStreamingHandler; 265 template <class ServiceType, class RequestType, class ResponseType> 266 friend class ::grpc::internal::ServerStreamingHandler; 267 template <class Streamer, bool WriteNeeded> 268 friend class ::grpc::internal::TemplatedBidiStreamingHandler; 269 template <StatusCode code> 270 friend class ::grpc::internal::ErrorMethodHandler; 271 friend class ::grpc::Server; 272 friend class ::grpc::ServerContext; 273 friend class ::grpc::ServerInterface; 274 template <class InputMessage, class OutputMessage> 275 friend class ::grpc::internal::BlockingUnaryCallImpl; 276 277 // Friends that need access to constructor for callback CQ 278 friend class ::grpc::Channel; 279 280 /// EXPERIMENTAL 281 /// Creates a Thread Local cache to store the first event 282 /// On this completion queue queued from this thread. Once 283 /// initialized, it must be flushed on the same thread. 284 class CompletionQueueTLSCache { 285 public: 286 CompletionQueueTLSCache(CompletionQueue* cq); 287 ~CompletionQueueTLSCache(); 288 bool Flush(void** tag, bool* ok); 289 290 private: 291 CompletionQueue* cq_; 292 bool flushed_; 293 }; 294 295 NextStatus AsyncNextInternal(void** tag, bool* ok, gpr_timespec deadline); 296 297 /// Wraps \a grpc_completion_queue_pluck. 298 /// \warning Must not be mixed with calls to \a Next. Pluck(internal::CompletionQueueTag * tag)299 bool Pluck(internal::CompletionQueueTag* tag) { 300 auto deadline = 301 g_core_codegen_interface->gpr_inf_future(GPR_CLOCK_REALTIME); 302 auto ev = g_core_codegen_interface->grpc_completion_queue_pluck( 303 cq_, tag, deadline, nullptr); 304 bool ok = ev.success != 0; 305 void* ignored = tag; 306 GPR_CODEGEN_ASSERT(tag->FinalizeResult(&ignored, &ok)); 307 GPR_CODEGEN_ASSERT(ignored == tag); 308 // Ignore mutations by FinalizeResult: Pluck returns the C API status 309 return ev.success != 0; 310 } 311 312 /// Performs a single polling pluck on \a tag. 313 /// \warning Must not be mixed with calls to \a Next. 314 /// 315 /// TODO: sreek - This calls tag->FinalizeResult() even if the cq_ is already 316 /// shutdown. This is most likely a bug and if it is a bug, then change this 317 /// implementation to simple call the other TryPluck function with a zero 318 /// timeout. i.e: 319 /// TryPluck(tag, gpr_time_0(GPR_CLOCK_REALTIME)) TryPluck(internal::CompletionQueueTag * tag)320 void TryPluck(internal::CompletionQueueTag* tag) { 321 auto deadline = g_core_codegen_interface->gpr_time_0(GPR_CLOCK_REALTIME); 322 auto ev = g_core_codegen_interface->grpc_completion_queue_pluck( 323 cq_, tag, deadline, nullptr); 324 if (ev.type == GRPC_QUEUE_TIMEOUT) return; 325 bool ok = ev.success != 0; 326 void* ignored = tag; 327 // the tag must be swallowed if using TryPluck 328 GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok)); 329 } 330 331 /// Performs a single polling pluck on \a tag. Calls tag->FinalizeResult if 332 /// the pluck() was successful and returned the tag. 333 /// 334 /// This exects tag->FinalizeResult (if called) to return 'false' i.e expects 335 /// that the tag is internal not something that is returned to the user. TryPluck(internal::CompletionQueueTag * tag,gpr_timespec deadline)336 void TryPluck(internal::CompletionQueueTag* tag, gpr_timespec deadline) { 337 auto ev = g_core_codegen_interface->grpc_completion_queue_pluck( 338 cq_, tag, deadline, nullptr); 339 if (ev.type == GRPC_QUEUE_TIMEOUT || ev.type == GRPC_QUEUE_SHUTDOWN) { 340 return; 341 } 342 343 bool ok = ev.success != 0; 344 void* ignored = tag; 345 GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok)); 346 } 347 348 /// Manage state of avalanching operations : completion queue tags that 349 /// trigger other completion queue operations. The underlying core completion 350 /// queue should not really shutdown until all avalanching operations have 351 /// been finalized. Note that we maintain the requirement that an avalanche 352 /// registration must take place before CQ shutdown (which must be maintained 353 /// elsehwere) InitialAvalanching()354 void InitialAvalanching() { 355 gpr_atm_rel_store(&avalanches_in_flight_, static_cast<gpr_atm>(1)); 356 } RegisterAvalanching()357 void RegisterAvalanching() { 358 gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_, 359 static_cast<gpr_atm>(1)); 360 } 361 void CompleteAvalanching(); 362 363 grpc_completion_queue* cq_; // owned 364 365 gpr_atm avalanches_in_flight_; 366 }; 367 368 /// A specific type of completion queue used by the processing of notifications 369 /// by servers. Instantiated by \a ServerBuilder. 370 class ServerCompletionQueue : public CompletionQueue { 371 public: IsFrequentlyPolled()372 bool IsFrequentlyPolled() { return polling_type_ != GRPC_CQ_NON_LISTENING; } 373 374 protected: 375 /// Default constructor ServerCompletionQueue()376 ServerCompletionQueue() : polling_type_(GRPC_CQ_DEFAULT_POLLING) {} 377 378 private: 379 /// \param is_frequently_polled Informs the GRPC library about whether the 380 /// server completion queue would be actively polled (by calling Next() or 381 /// AsyncNext()). By default all server completion queues are assumed to be 382 /// frequently polled. ServerCompletionQueue(grpc_cq_polling_type polling_type)383 ServerCompletionQueue(grpc_cq_polling_type polling_type) 384 : CompletionQueue(grpc_completion_queue_attributes{ 385 GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, polling_type, nullptr}), 386 polling_type_(polling_type) {} 387 388 grpc_cq_polling_type polling_type_; 389 friend class ServerBuilder; 390 }; 391 392 } // namespace grpc 393 394 #endif // GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_H 395