• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Generic wait-for-completion handler;
3  *
4  * It differs from semaphores in that their default case is the opposite,
5  * wait_for_completion default blocks whereas semaphore default non-block. The
6  * interface also makes it easy to 'complete' multiple waiting threads,
7  * something which isn't entirely natural for semaphores.
8  *
9  * But more importantly, the primitive documents the usage. Semaphores would
10  * typically be used for exclusion which gives rise to priority inversion.
11  * Waiting for completion is a typically sync point, but not an exclusion point.
12  */
13 
14 #include <linux/sched.h>
15 #include <linux/completion.h>
16 
17 /**
18  * complete: - signals a single thread waiting on this completion
19  * @x:  holds the state of this particular completion
20  *
21  * This will wake up a single thread waiting on this completion. Threads will be
22  * awakened in the same order in which they were queued.
23  *
24  * See also complete_all(), wait_for_completion() and related routines.
25  *
26  * It may be assumed that this function implies a write memory barrier before
27  * changing the task state if and only if any tasks are woken up.
28  */
complete(struct completion * x)29 void complete(struct completion *x)
30 {
31 	unsigned long flags;
32 
33 	spin_lock_irqsave(&x->wait.lock, flags);
34 	x->done++;
35 	__wake_up_locked(&x->wait, TASK_NORMAL, 1);
36 	spin_unlock_irqrestore(&x->wait.lock, flags);
37 }
38 EXPORT_SYMBOL(complete);
39 
40 /**
41  * complete_all: - signals all threads waiting on this completion
42  * @x:  holds the state of this particular completion
43  *
44  * This will wake up all threads waiting on this particular completion event.
45  *
46  * It may be assumed that this function implies a write memory barrier before
47  * changing the task state if and only if any tasks are woken up.
48  */
complete_all(struct completion * x)49 void complete_all(struct completion *x)
50 {
51 	unsigned long flags;
52 
53 	spin_lock_irqsave(&x->wait.lock, flags);
54 	x->done += UINT_MAX/2;
55 	__wake_up_locked(&x->wait, TASK_NORMAL, 0);
56 	spin_unlock_irqrestore(&x->wait.lock, flags);
57 }
58 EXPORT_SYMBOL(complete_all);
59 
60 static inline long __sched
do_wait_for_common(struct completion * x,long (* action)(long),long timeout,int state)61 do_wait_for_common(struct completion *x,
62 		   long (*action)(long), long timeout, int state)
63 {
64 	if (!x->done) {
65 		DECLARE_WAITQUEUE(wait, current);
66 
67 		__add_wait_queue_tail_exclusive(&x->wait, &wait);
68 		do {
69 			if (signal_pending_state(state, current)) {
70 				timeout = -ERESTARTSYS;
71 				break;
72 			}
73 			__set_current_state(state);
74 			spin_unlock_irq(&x->wait.lock);
75 			timeout = action(timeout);
76 			spin_lock_irq(&x->wait.lock);
77 		} while (!x->done && timeout);
78 		__remove_wait_queue(&x->wait, &wait);
79 		if (!x->done)
80 			return timeout;
81 	}
82 	x->done--;
83 	return timeout ?: 1;
84 }
85 
86 static inline long __sched
__wait_for_common(struct completion * x,long (* action)(long),long timeout,int state)87 __wait_for_common(struct completion *x,
88 		  long (*action)(long), long timeout, int state)
89 {
90 	might_sleep();
91 
92 	spin_lock_irq(&x->wait.lock);
93 	timeout = do_wait_for_common(x, action, timeout, state);
94 	spin_unlock_irq(&x->wait.lock);
95 	return timeout;
96 }
97 
98 static long __sched
wait_for_common(struct completion * x,long timeout,int state)99 wait_for_common(struct completion *x, long timeout, int state)
100 {
101 	return __wait_for_common(x, schedule_timeout, timeout, state);
102 }
103 
104 static long __sched
wait_for_common_io(struct completion * x,long timeout,int state)105 wait_for_common_io(struct completion *x, long timeout, int state)
106 {
107 	return __wait_for_common(x, io_schedule_timeout, timeout, state);
108 }
109 
110 /**
111  * wait_for_completion: - waits for completion of a task
112  * @x:  holds the state of this particular completion
113  *
114  * This waits to be signaled for completion of a specific task. It is NOT
115  * interruptible and there is no timeout.
116  *
117  * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
118  * and interrupt capability. Also see complete().
119  */
wait_for_completion(struct completion * x)120 void __sched wait_for_completion(struct completion *x)
121 {
122 	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
123 }
124 EXPORT_SYMBOL(wait_for_completion);
125 
126 /**
127  * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
128  * @x:  holds the state of this particular completion
129  * @timeout:  timeout value in jiffies
130  *
131  * This waits for either a completion of a specific task to be signaled or for a
132  * specified timeout to expire. The timeout is in jiffies. It is not
133  * interruptible.
134  *
135  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
136  * till timeout) if completed.
137  */
138 unsigned long __sched
wait_for_completion_timeout(struct completion * x,unsigned long timeout)139 wait_for_completion_timeout(struct completion *x, unsigned long timeout)
140 {
141 	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
142 }
143 EXPORT_SYMBOL(wait_for_completion_timeout);
144 
145 /**
146  * wait_for_completion_io: - waits for completion of a task
147  * @x:  holds the state of this particular completion
148  *
149  * This waits to be signaled for completion of a specific task. It is NOT
150  * interruptible and there is no timeout. The caller is accounted as waiting
151  * for IO (which traditionally means blkio only).
152  */
wait_for_completion_io(struct completion * x)153 void __sched wait_for_completion_io(struct completion *x)
154 {
155 	wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
156 }
157 EXPORT_SYMBOL(wait_for_completion_io);
158 
159 /**
160  * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
161  * @x:  holds the state of this particular completion
162  * @timeout:  timeout value in jiffies
163  *
164  * This waits for either a completion of a specific task to be signaled or for a
165  * specified timeout to expire. The timeout is in jiffies. It is not
166  * interruptible. The caller is accounted as waiting for IO (which traditionally
167  * means blkio only).
168  *
169  * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
170  * till timeout) if completed.
171  */
172 unsigned long __sched
wait_for_completion_io_timeout(struct completion * x,unsigned long timeout)173 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
174 {
175 	return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
176 }
177 EXPORT_SYMBOL(wait_for_completion_io_timeout);
178 
179 /**
180  * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
181  * @x:  holds the state of this particular completion
182  *
183  * This waits for completion of a specific task to be signaled. It is
184  * interruptible.
185  *
186  * Return: -ERESTARTSYS if interrupted, 0 if completed.
187  */
wait_for_completion_interruptible(struct completion * x)188 int __sched wait_for_completion_interruptible(struct completion *x)
189 {
190 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
191 	if (t == -ERESTARTSYS)
192 		return t;
193 	return 0;
194 }
195 EXPORT_SYMBOL(wait_for_completion_interruptible);
196 
197 /**
198  * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
199  * @x:  holds the state of this particular completion
200  * @timeout:  timeout value in jiffies
201  *
202  * This waits for either a completion of a specific task to be signaled or for a
203  * specified timeout to expire. It is interruptible. The timeout is in jiffies.
204  *
205  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
206  * or number of jiffies left till timeout) if completed.
207  */
208 long __sched
wait_for_completion_interruptible_timeout(struct completion * x,unsigned long timeout)209 wait_for_completion_interruptible_timeout(struct completion *x,
210 					  unsigned long timeout)
211 {
212 	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
213 }
214 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
215 
216 /**
217  * wait_for_completion_killable: - waits for completion of a task (killable)
218  * @x:  holds the state of this particular completion
219  *
220  * This waits to be signaled for completion of a specific task. It can be
221  * interrupted by a kill signal.
222  *
223  * Return: -ERESTARTSYS if interrupted, 0 if completed.
224  */
wait_for_completion_killable(struct completion * x)225 int __sched wait_for_completion_killable(struct completion *x)
226 {
227 	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
228 	if (t == -ERESTARTSYS)
229 		return t;
230 	return 0;
231 }
232 EXPORT_SYMBOL(wait_for_completion_killable);
233 
234 /**
235  * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
236  * @x:  holds the state of this particular completion
237  * @timeout:  timeout value in jiffies
238  *
239  * This waits for either a completion of a specific task to be
240  * signaled or for a specified timeout to expire. It can be
241  * interrupted by a kill signal. The timeout is in jiffies.
242  *
243  * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
244  * or number of jiffies left till timeout) if completed.
245  */
246 long __sched
wait_for_completion_killable_timeout(struct completion * x,unsigned long timeout)247 wait_for_completion_killable_timeout(struct completion *x,
248 				     unsigned long timeout)
249 {
250 	return wait_for_common(x, timeout, TASK_KILLABLE);
251 }
252 EXPORT_SYMBOL(wait_for_completion_killable_timeout);
253 
254 /**
255  *	try_wait_for_completion - try to decrement a completion without blocking
256  *	@x:	completion structure
257  *
258  *	Return: 0 if a decrement cannot be done without blocking
259  *		 1 if a decrement succeeded.
260  *
261  *	If a completion is being used as a counting completion,
262  *	attempt to decrement the counter without blocking. This
263  *	enables us to avoid waiting if the resource the completion
264  *	is protecting is not available.
265  */
try_wait_for_completion(struct completion * x)266 bool try_wait_for_completion(struct completion *x)
267 {
268 	unsigned long flags;
269 	int ret = 1;
270 
271 	/*
272 	 * Since x->done will need to be locked only
273 	 * in the non-blocking case, we check x->done
274 	 * first without taking the lock so we can
275 	 * return early in the blocking case.
276 	 */
277 	if (!READ_ONCE(x->done))
278 		return 0;
279 
280 	spin_lock_irqsave(&x->wait.lock, flags);
281 	if (!x->done)
282 		ret = 0;
283 	else
284 		x->done--;
285 	spin_unlock_irqrestore(&x->wait.lock, flags);
286 	return ret;
287 }
288 EXPORT_SYMBOL(try_wait_for_completion);
289 
290 /**
291  *	completion_done - Test to see if a completion has any waiters
292  *	@x:	completion structure
293  *
294  *	Return: 0 if there are waiters (wait_for_completion() in progress)
295  *		 1 if there are no waiters.
296  *
297  */
completion_done(struct completion * x)298 bool completion_done(struct completion *x)
299 {
300 	if (!READ_ONCE(x->done))
301 		return false;
302 
303 	/*
304 	 * If ->done, we need to wait for complete() to release ->wait.lock
305 	 * otherwise we can end up freeing the completion before complete()
306 	 * is done referencing it.
307 	 *
308 	 * The RMB pairs with complete()'s RELEASE of ->wait.lock and orders
309 	 * the loads of ->done and ->wait.lock such that we cannot observe
310 	 * the lock before complete() acquires it while observing the ->done
311 	 * after it's acquired the lock.
312 	 */
313 	smp_rmb();
314 	spin_unlock_wait(&x->wait.lock);
315 	return true;
316 }
317 EXPORT_SYMBOL(completion_done);
318