1 //
2 //
3 // Copyright 2017 gRPC authors.
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License");
6 // you may not use this file except in compliance with the License.
7 // You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
17 //
18
19 #include "src/core/lib/iomgr/lockfree_event.h"
20
21 #include <grpc/support/port_platform.h>
22
23 #include "absl/log/check.h"
24 #include "absl/log/log.h"
25 #include "src/core/lib/debug/trace.h"
26 #include "src/core/lib/iomgr/exec_ctx.h"
27 #include "src/core/util/crash.h"
28
29 // 'state' holds the to call when the fd is readable or writable respectively.
30 // It can contain one of the following values:
31 // kClosureReady : The fd has an I/O event of interest but there is no
32 // closure yet to execute
33
34 // kClosureNotReady : The fd has no I/O event of interest
35
36 // closure ptr : The closure to be executed when the fd has an I/O
37 // event of interest
38
39 // shutdown_error | kShutdownBit :
40 // 'shutdown_error' field ORed with kShutdownBit.
41 // This indicates that the fd is shutdown. Since all
42 // memory allocations are word-aligned, the lower two
43 // bits of the shutdown_error pointer are always 0. So
44 // it is safe to OR these with kShutdownBit
45
46 // Valid state transitions:
47
48 // <closure ptr> <-----3------ kClosureNotReady -----1-------> kClosureReady
49 // | | ^ | ^ | |
50 // | | | | | | |
51 // | +--------------4----------+ 6 +---------2---------------+ |
52 // | | |
53 // | v |
54 // +-----5-------> [shutdown_error | kShutdownBit] <-------7---------+
55
56 // For 1, 4 : See SetReady() function
57 // For 2, 3 : See NotifyOn() function
58 // For 5,6,7: See SetShutdown() function
59
60 namespace grpc_core {
61
LockfreeEvent()62 LockfreeEvent::LockfreeEvent() { InitEvent(); }
63
InitEvent()64 void LockfreeEvent::InitEvent() {
65 // Perform an atomic store to start the state machine.
66
67 // Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
68 // state, while a file descriptor is on a freelist. In such a state it may
69 // be SetReady'd, and so we need to perform an atomic operation here to
70 // ensure no races
71 gpr_atm_no_barrier_store(&state_, kClosureNotReady);
72 }
73
DestroyEvent()74 void LockfreeEvent::DestroyEvent() {
75 gpr_atm curr;
76 do {
77 curr = gpr_atm_no_barrier_load(&state_);
78 if (curr & kShutdownBit) {
79 internal::StatusFreeHeapPtr(curr & ~kShutdownBit);
80 } else {
81 CHECK(curr == kClosureNotReady || curr == kClosureReady);
82 }
83 // we CAS in a shutdown, no error value here. If this event is interacted
84 // with post-deletion (see the note in the constructor) we want the bit
85 // pattern to prevent error retention in a deleted object
86 } while (!gpr_atm_no_barrier_cas(&state_, curr,
87 kShutdownBit /* shutdown, no error */));
88 }
89
NotifyOn(grpc_closure * closure)90 void LockfreeEvent::NotifyOn(grpc_closure* closure) {
91 while (true) {
92 // This load needs to be an acquire load because this can be a shutdown
93 // error that we might need to reference. Adding acquire semantics makes
94 // sure that the shutdown error has been initialized properly before us
95 // referencing it.
96 gpr_atm curr = gpr_atm_acq_load(&state_);
97 GRPC_TRACE_VLOG(polling, 2) << "LockfreeEvent::NotifyOn: " << this
98 << " curr=" << curr << " closure=" << closure;
99 switch (curr) {
100 case kClosureNotReady: {
101 // kClosureNotReady -> <closure>.
102
103 // We're guaranteed by API that there's an acquire barrier before here,
104 // so there's no need to double-dip and this can be a release-only.
105
106 // The release itself pairs with the acquire half of a set_ready full
107 // barrier.
108 if (gpr_atm_rel_cas(&state_, kClosureNotReady,
109 reinterpret_cast<gpr_atm>(closure))) {
110 return; // Successful. Return
111 }
112
113 break; // retry
114 }
115
116 case kClosureReady: {
117 // Change the state to kClosureNotReady. Schedule the closure if
118 // successful. If not, the state most likely transitioned to shutdown.
119 // We should retry.
120
121 // This can be a no-barrier cas since the state is being transitioned to
122 // kClosureNotReady; set_ready and set_shutdown do not schedule any
123 // closure when transitioning out of CLOSURE_NO_READY state (i.e there
124 // is no other code that needs to 'happen-after' this)
125 if (gpr_atm_no_barrier_cas(&state_, kClosureReady, kClosureNotReady)) {
126 ExecCtx::Run(DEBUG_LOCATION, closure, absl::OkStatus());
127 return; // Successful. Return
128 }
129
130 break; // retry
131 }
132
133 default: {
134 // 'curr' is either a closure or the fd is shutdown(in which case 'curr'
135 // contains a pointer to the shutdown-error). If the fd is shutdown,
136 // schedule the closure with the shutdown error
137 if ((curr & kShutdownBit) > 0) {
138 grpc_error_handle shutdown_err =
139 internal::StatusGetFromHeapPtr(curr & ~kShutdownBit);
140 ExecCtx::Run(
141 DEBUG_LOCATION, closure,
142 GRPC_ERROR_CREATE_REFERENCING("FD Shutdown", &shutdown_err, 1));
143 return;
144 }
145
146 // There is already a closure!. This indicates a bug in the code
147 Crash(
148 "LockfreeEvent::NotifyOn: notify_on called with a previous "
149 "callback still pending");
150 }
151 }
152 }
153
154 GPR_UNREACHABLE_CODE(return);
155 }
156
SetShutdown(grpc_error_handle shutdown_error)157 bool LockfreeEvent::SetShutdown(grpc_error_handle shutdown_error) {
158 intptr_t status_ptr = internal::StatusAllocHeapPtr(shutdown_error);
159 gpr_atm new_state = status_ptr | kShutdownBit;
160
161 while (true) {
162 gpr_atm curr = gpr_atm_no_barrier_load(&state_);
163 GRPC_TRACE_VLOG(polling, 2)
164 << "LockfreeEvent::SetShutdown: " << &state_ << " curr=" << curr
165 << " err=" << StatusToString(shutdown_error);
166 switch (curr) {
167 case kClosureReady:
168 case kClosureNotReady:
169 // Need a full barrier here so that the initial load in notify_on
170 // doesn't need a barrier
171 if (gpr_atm_full_cas(&state_, curr, new_state)) {
172 return true; // early out
173 }
174 break; // retry
175
176 default: {
177 // 'curr' is either a closure or the fd is already shutdown
178
179 // If fd is already shutdown, we are done
180 if ((curr & kShutdownBit) > 0) {
181 internal::StatusFreeHeapPtr(status_ptr);
182 return false;
183 }
184
185 // Fd is not shutdown. Schedule the closure and move the state to
186 // shutdown state.
187 // Needs an acquire to pair with setting the closure (and get a
188 // happens-after on that edge), and a release to pair with anything
189 // loading the shutdown state.
190 if (gpr_atm_full_cas(&state_, curr, new_state)) {
191 ExecCtx::Run(
192 DEBUG_LOCATION, reinterpret_cast<grpc_closure*>(curr),
193 GRPC_ERROR_CREATE_REFERENCING("FD Shutdown", &shutdown_error, 1));
194 return true;
195 }
196
197 // 'curr' was a closure but now changed to a different state. We will
198 // have to retry
199 break;
200 }
201 }
202 }
203
204 GPR_UNREACHABLE_CODE(return false);
205 }
206
SetReady()207 void LockfreeEvent::SetReady() {
208 while (true) {
209 gpr_atm curr = gpr_atm_no_barrier_load(&state_);
210
211 GRPC_TRACE_VLOG(polling, 2)
212 << "LockfreeEvent::SetReady: " << &state_ << " curr=" << curr;
213
214 switch (curr) {
215 case kClosureReady: {
216 // Already ready. We are done here
217 return;
218 }
219
220 case kClosureNotReady: {
221 // No barrier required as we're transitioning to a state that does not
222 // involve a closure
223 if (gpr_atm_no_barrier_cas(&state_, kClosureNotReady, kClosureReady)) {
224 return; // early out
225 }
226 break; // retry
227 }
228
229 default: {
230 // 'curr' is either a closure or the fd is shutdown
231 if ((curr & kShutdownBit) > 0) {
232 // The fd is shutdown. Do nothing
233 return;
234 }
235 // Full cas: acquire pairs with this cas' release in the event of a
236 // spurious set_ready; release pairs with this or the acquire in
237 // notify_on (or set_shutdown)
238 else if (gpr_atm_full_cas(&state_, curr, kClosureNotReady)) {
239 ExecCtx::Run(DEBUG_LOCATION, reinterpret_cast<grpc_closure*>(curr),
240 absl::OkStatus());
241 return;
242 }
243 // else the state changed again (only possible by either a racing
244 // set_ready or set_shutdown functions. In both these cases, the closure
245 // would have been scheduled for execution. So we are done here
246 return;
247 }
248 }
249 }
250 }
251
252 } // namespace grpc_core
253