1 /*
2 *
3 * Copyright 2017 gRPC authors.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 *
17 */
18
19 #include <grpc/support/port_platform.h>
20
21 #include "src/core/lib/iomgr/lockfree_event.h"
22
23 #include <grpc/support/log.h>
24
25 #include "src/core/lib/debug/trace.h"
26
27 extern grpc_core::TraceFlag grpc_polling_trace;
28
29 /* 'state' holds the to call when the fd is readable or writable respectively.
30 It can contain one of the following values:
31 kClosureReady : The fd has an I/O event of interest but there is no
32 closure yet to execute
33
34 kClosureNotReady : The fd has no I/O event of interest
35
36 closure ptr : The closure to be executed when the fd has an I/O
37 event of interest
38
39 shutdown_error | kShutdownBit :
40 'shutdown_error' field ORed with kShutdownBit.
41 This indicates that the fd is shutdown. Since all
42 memory allocations are word-aligned, the lower two
43 bits of the shutdown_error pointer are always 0. So
44 it is safe to OR these with kShutdownBit
45
46 Valid state transitions:
47
48 <closure ptr> <-----3------ kClosureNotReady -----1-------> kClosureReady
49 | | ^ | ^ | |
50 | | | | | | |
51 | +--------------4----------+ 6 +---------2---------------+ |
52 | | |
53 | v |
54 +-----5-------> [shutdown_error | kShutdownBit] <-------7---------+
55
56 For 1, 4 : See SetReady() function
57 For 2, 3 : See NotifyOn() function
58 For 5,6,7: See SetShutdown() function */
59
60 namespace grpc_core {
61
LockfreeEvent()62 LockfreeEvent::LockfreeEvent() { InitEvent(); }
63
InitEvent()64 void LockfreeEvent::InitEvent() {
65 /* Perform an atomic store to start the state machine.
66
67 Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
68 state, while a file descriptor is on a freelist. In such a state it may
69 be SetReady'd, and so we need to perform an atomic operation here to
70 ensure no races */
71 gpr_atm_no_barrier_store(&state_, kClosureNotReady);
72 }
73
DestroyEvent()74 void LockfreeEvent::DestroyEvent() {
75 gpr_atm curr;
76 do {
77 curr = gpr_atm_no_barrier_load(&state_);
78 if (curr & kShutdownBit) {
79 GRPC_ERROR_UNREF((grpc_error*)(curr & ~kShutdownBit));
80 } else {
81 GPR_ASSERT(curr == kClosureNotReady || curr == kClosureReady);
82 }
83 /* we CAS in a shutdown, no error value here. If this event is interacted
84 with post-deletion (see the note in the constructor) we want the bit
85 pattern to prevent error retention in a deleted object */
86 } while (!gpr_atm_no_barrier_cas(&state_, curr,
87 kShutdownBit /* shutdown, no error */));
88 }
89
NotifyOn(grpc_closure * closure)90 void LockfreeEvent::NotifyOn(grpc_closure* closure) {
91 while (true) {
92 /* This load needs to be an acquire load because this can be a shutdown
93 * error that we might need to reference. Adding acquire semantics makes
94 * sure that the shutdown error has been initialized properly before us
95 * referencing it. */
96 gpr_atm curr = gpr_atm_acq_load(&state_);
97 if (grpc_polling_trace.enabled()) {
98 gpr_log(GPR_ERROR, "LockfreeEvent::NotifyOn: %p curr=%p closure=%p", this,
99 (void*)curr, closure);
100 }
101 switch (curr) {
102 case kClosureNotReady: {
103 /* kClosureNotReady -> <closure>.
104
105 We're guaranteed by API that there's an acquire barrier before here,
106 so there's no need to double-dip and this can be a release-only.
107
108 The release itself pairs with the acquire half of a set_ready full
109 barrier. */
110 if (gpr_atm_rel_cas(&state_, kClosureNotReady, (gpr_atm)closure)) {
111 return; /* Successful. Return */
112 }
113
114 break; /* retry */
115 }
116
117 case kClosureReady: {
118 /* Change the state to kClosureNotReady. Schedule the closure if
119 successful. If not, the state most likely transitioned to shutdown.
120 We should retry.
121
122 This can be a no-barrier cas since the state is being transitioned to
123 kClosureNotReady; set_ready and set_shutdown do not schedule any
124 closure when transitioning out of CLOSURE_NO_READY state (i.e there
125 is no other code that needs to 'happen-after' this) */
126 if (gpr_atm_no_barrier_cas(&state_, kClosureReady, kClosureNotReady)) {
127 GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_NONE);
128 return; /* Successful. Return */
129 }
130
131 break; /* retry */
132 }
133
134 default: {
135 /* 'curr' is either a closure or the fd is shutdown(in which case 'curr'
136 contains a pointer to the shutdown-error). If the fd is shutdown,
137 schedule the closure with the shutdown error */
138 if ((curr & kShutdownBit) > 0) {
139 grpc_error* shutdown_err = (grpc_error*)(curr & ~kShutdownBit);
140 GRPC_CLOSURE_SCHED(closure,
141 GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
142 "FD Shutdown", &shutdown_err, 1));
143 return;
144 }
145
146 /* There is already a closure!. This indicates a bug in the code */
147 gpr_log(GPR_ERROR,
148 "LockfreeEvent::NotifyOn: notify_on called with a previous "
149 "callback still pending");
150 abort();
151 }
152 }
153 }
154
155 GPR_UNREACHABLE_CODE(return );
156 }
157
SetShutdown(grpc_error * shutdown_err)158 bool LockfreeEvent::SetShutdown(grpc_error* shutdown_err) {
159 gpr_atm new_state = (gpr_atm)shutdown_err | kShutdownBit;
160
161 while (true) {
162 gpr_atm curr = gpr_atm_no_barrier_load(&state_);
163 if (grpc_polling_trace.enabled()) {
164 gpr_log(GPR_ERROR, "LockfreeEvent::SetShutdown: %p curr=%p err=%s",
165 &state_, (void*)curr, grpc_error_string(shutdown_err));
166 }
167 switch (curr) {
168 case kClosureReady:
169 case kClosureNotReady:
170 /* Need a full barrier here so that the initial load in notify_on
171 doesn't need a barrier */
172 if (gpr_atm_full_cas(&state_, curr, new_state)) {
173 return true; /* early out */
174 }
175 break; /* retry */
176
177 default: {
178 /* 'curr' is either a closure or the fd is already shutdown */
179
180 /* If fd is already shutdown, we are done */
181 if ((curr & kShutdownBit) > 0) {
182 GRPC_ERROR_UNREF(shutdown_err);
183 return false;
184 }
185
186 /* Fd is not shutdown. Schedule the closure and move the state to
187 shutdown state.
188 Needs an acquire to pair with setting the closure (and get a
189 happens-after on that edge), and a release to pair with anything
190 loading the shutdown state. */
191 if (gpr_atm_full_cas(&state_, curr, new_state)) {
192 GRPC_CLOSURE_SCHED((grpc_closure*)curr,
193 GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
194 "FD Shutdown", &shutdown_err, 1));
195 return true;
196 }
197
198 /* 'curr' was a closure but now changed to a different state. We will
199 have to retry */
200 break;
201 }
202 }
203 }
204
205 GPR_UNREACHABLE_CODE(return false);
206 }
207
SetReady()208 void LockfreeEvent::SetReady() {
209 while (true) {
210 gpr_atm curr = gpr_atm_no_barrier_load(&state_);
211
212 if (grpc_polling_trace.enabled()) {
213 gpr_log(GPR_ERROR, "LockfreeEvent::SetReady: %p curr=%p", &state_,
214 (void*)curr);
215 }
216
217 switch (curr) {
218 case kClosureReady: {
219 /* Already ready. We are done here */
220 return;
221 }
222
223 case kClosureNotReady: {
224 /* No barrier required as we're transitioning to a state that does not
225 involve a closure */
226 if (gpr_atm_no_barrier_cas(&state_, kClosureNotReady, kClosureReady)) {
227 return; /* early out */
228 }
229 break; /* retry */
230 }
231
232 default: {
233 /* 'curr' is either a closure or the fd is shutdown */
234 if ((curr & kShutdownBit) > 0) {
235 /* The fd is shutdown. Do nothing */
236 return;
237 }
238 /* Full cas: acquire pairs with this cas' release in the event of a
239 spurious set_ready; release pairs with this or the acquire in
240 notify_on (or set_shutdown) */
241 else if (gpr_atm_full_cas(&state_, curr, kClosureNotReady)) {
242 GRPC_CLOSURE_SCHED((grpc_closure*)curr, GRPC_ERROR_NONE);
243 return;
244 }
245 /* else the state changed again (only possible by either a racing
246 set_ready or set_shutdown functions. In both these cases, the closure
247 would have been scheduled for execution. So we are done here */
248 return;
249 }
250 }
251 }
252 }
253
254 } // namespace grpc_core
255