1 /*
2 * Copyright (2004) Linus Torvalds
3 *
4 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5 *
6 * Copyright (2004, 2005) Ingo Molnar
7 *
8 * This file contains the spinlock/rwlock implementations for the
9 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10 *
11 * Note that some architectures have special knowledge about the
12 * stack frames of these functions in their profile_pc. If you
13 * change anything significant here that could change the stack
14 * frame contact the architecture maintainers.
15 */
16
17 #include <linux/linkage.h>
18 #include <linux/preempt.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/debug_locks.h>
22 #include <linux/module.h>
23
_spin_trylock(spinlock_t * lock)24 int __lockfunc _spin_trylock(spinlock_t *lock)
25 {
26 preempt_disable();
27 if (_raw_spin_trylock(lock)) {
28 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
29 return 1;
30 }
31
32 preempt_enable();
33 return 0;
34 }
35 EXPORT_SYMBOL(_spin_trylock);
36
_read_trylock(rwlock_t * lock)37 int __lockfunc _read_trylock(rwlock_t *lock)
38 {
39 preempt_disable();
40 if (_raw_read_trylock(lock)) {
41 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
42 return 1;
43 }
44
45 preempt_enable();
46 return 0;
47 }
48 EXPORT_SYMBOL(_read_trylock);
49
_write_trylock(rwlock_t * lock)50 int __lockfunc _write_trylock(rwlock_t *lock)
51 {
52 preempt_disable();
53 if (_raw_write_trylock(lock)) {
54 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
55 return 1;
56 }
57
58 preempt_enable();
59 return 0;
60 }
61 EXPORT_SYMBOL(_write_trylock);
62
63 /*
64 * If lockdep is enabled then we use the non-preemption spin-ops
65 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67 */
68 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
69
_read_lock(rwlock_t * lock)70 void __lockfunc _read_lock(rwlock_t *lock)
71 {
72 preempt_disable();
73 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
74 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
75 }
76 EXPORT_SYMBOL(_read_lock);
77
_spin_lock_irqsave(spinlock_t * lock)78 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
79 {
80 unsigned long flags;
81
82 local_irq_save(flags);
83 preempt_disable();
84 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
85 /*
86 * On lockdep we dont want the hand-coded irq-enable of
87 * _raw_spin_lock_flags() code, because lockdep assumes
88 * that interrupts are not re-enabled during lock-acquire:
89 */
90 #ifdef CONFIG_LOCKDEP
91 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
92 #else
93 _raw_spin_lock_flags(lock, &flags);
94 #endif
95 return flags;
96 }
97 EXPORT_SYMBOL(_spin_lock_irqsave);
98
_spin_lock_irq(spinlock_t * lock)99 void __lockfunc _spin_lock_irq(spinlock_t *lock)
100 {
101 local_irq_disable();
102 preempt_disable();
103 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
104 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
105 }
106 EXPORT_SYMBOL(_spin_lock_irq);
107
_spin_lock_bh(spinlock_t * lock)108 void __lockfunc _spin_lock_bh(spinlock_t *lock)
109 {
110 local_bh_disable();
111 preempt_disable();
112 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
113 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
114 }
115 EXPORT_SYMBOL(_spin_lock_bh);
116
_read_lock_irqsave(rwlock_t * lock)117 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
118 {
119 unsigned long flags;
120
121 local_irq_save(flags);
122 preempt_disable();
123 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
124 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
125 return flags;
126 }
127 EXPORT_SYMBOL(_read_lock_irqsave);
128
_read_lock_irq(rwlock_t * lock)129 void __lockfunc _read_lock_irq(rwlock_t *lock)
130 {
131 local_irq_disable();
132 preempt_disable();
133 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
134 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
135 }
136 EXPORT_SYMBOL(_read_lock_irq);
137
_read_lock_bh(rwlock_t * lock)138 void __lockfunc _read_lock_bh(rwlock_t *lock)
139 {
140 local_bh_disable();
141 preempt_disable();
142 rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
143 LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
144 }
145 EXPORT_SYMBOL(_read_lock_bh);
146
_write_lock_irqsave(rwlock_t * lock)147 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
148 {
149 unsigned long flags;
150
151 local_irq_save(flags);
152 preempt_disable();
153 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
154 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
155 return flags;
156 }
157 EXPORT_SYMBOL(_write_lock_irqsave);
158
_write_lock_irq(rwlock_t * lock)159 void __lockfunc _write_lock_irq(rwlock_t *lock)
160 {
161 local_irq_disable();
162 preempt_disable();
163 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
164 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
165 }
166 EXPORT_SYMBOL(_write_lock_irq);
167
_write_lock_bh(rwlock_t * lock)168 void __lockfunc _write_lock_bh(rwlock_t *lock)
169 {
170 local_bh_disable();
171 preempt_disable();
172 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
173 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
174 }
175 EXPORT_SYMBOL(_write_lock_bh);
176
_spin_lock(spinlock_t * lock)177 void __lockfunc _spin_lock(spinlock_t *lock)
178 {
179 preempt_disable();
180 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
181 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
182 }
183
184 EXPORT_SYMBOL(_spin_lock);
185
_write_lock(rwlock_t * lock)186 void __lockfunc _write_lock(rwlock_t *lock)
187 {
188 preempt_disable();
189 rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
190 LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
191 }
192
193 EXPORT_SYMBOL(_write_lock);
194
195 #else /* CONFIG_PREEMPT: */
196
197 /*
198 * This could be a long-held lock. We both prepare to spin for a long
199 * time (making _this_ CPU preemptable if possible), and we also signal
200 * towards that other CPU that it should break the lock ASAP.
201 *
202 * (We do this in a function because inlining it would be excessive.)
203 */
204
205 #define BUILD_LOCK_OPS(op, locktype) \
206 void __lockfunc _##op##_lock(locktype##_t *lock) \
207 { \
208 for (;;) { \
209 preempt_disable(); \
210 if (likely(_raw_##op##_trylock(lock))) \
211 break; \
212 preempt_enable(); \
213 \
214 if (!(lock)->break_lock) \
215 (lock)->break_lock = 1; \
216 while (!op##_can_lock(lock) && (lock)->break_lock) \
217 _raw_##op##_relax(&lock->raw_lock); \
218 } \
219 (lock)->break_lock = 0; \
220 } \
221 \
222 EXPORT_SYMBOL(_##op##_lock); \
223 \
224 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
225 { \
226 unsigned long flags; \
227 \
228 for (;;) { \
229 preempt_disable(); \
230 local_irq_save(flags); \
231 if (likely(_raw_##op##_trylock(lock))) \
232 break; \
233 local_irq_restore(flags); \
234 preempt_enable(); \
235 \
236 if (!(lock)->break_lock) \
237 (lock)->break_lock = 1; \
238 while (!op##_can_lock(lock) && (lock)->break_lock) \
239 _raw_##op##_relax(&lock->raw_lock); \
240 } \
241 (lock)->break_lock = 0; \
242 return flags; \
243 } \
244 \
245 EXPORT_SYMBOL(_##op##_lock_irqsave); \
246 \
247 void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
248 { \
249 _##op##_lock_irqsave(lock); \
250 } \
251 \
252 EXPORT_SYMBOL(_##op##_lock_irq); \
253 \
254 void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
255 { \
256 unsigned long flags; \
257 \
258 /* */ \
259 /* Careful: we must exclude softirqs too, hence the */ \
260 /* irq-disabling. We use the generic preemption-aware */ \
261 /* function: */ \
262 /**/ \
263 flags = _##op##_lock_irqsave(lock); \
264 local_bh_disable(); \
265 local_irq_restore(flags); \
266 } \
267 \
268 EXPORT_SYMBOL(_##op##_lock_bh)
269
270 /*
271 * Build preemption-friendly versions of the following
272 * lock-spinning functions:
273 *
274 * _[spin|read|write]_lock()
275 * _[spin|read|write]_lock_irq()
276 * _[spin|read|write]_lock_irqsave()
277 * _[spin|read|write]_lock_bh()
278 */
279 BUILD_LOCK_OPS(spin, spinlock);
280 BUILD_LOCK_OPS(read, rwlock);
281 BUILD_LOCK_OPS(write, rwlock);
282
283 #endif /* CONFIG_PREEMPT */
284
285 #ifdef CONFIG_DEBUG_LOCK_ALLOC
286
_spin_lock_nested(spinlock_t * lock,int subclass)287 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
288 {
289 preempt_disable();
290 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
291 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
292 }
293 EXPORT_SYMBOL(_spin_lock_nested);
294
_spin_lock_irqsave_nested(spinlock_t * lock,int subclass)295 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
296 {
297 unsigned long flags;
298
299 local_irq_save(flags);
300 preempt_disable();
301 spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
302 /*
303 * On lockdep we dont want the hand-coded irq-enable of
304 * _raw_spin_lock_flags() code, because lockdep assumes
305 * that interrupts are not re-enabled during lock-acquire:
306 */
307 #ifdef CONFIG_LOCKDEP
308 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
309 #else
310 _raw_spin_lock_flags(lock, &flags);
311 #endif
312 return flags;
313 }
314 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
315
_spin_lock_nest_lock(spinlock_t * lock,struct lockdep_map * nest_lock)316 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
317 struct lockdep_map *nest_lock)
318 {
319 preempt_disable();
320 spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
321 LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
322 }
323 EXPORT_SYMBOL(_spin_lock_nest_lock);
324
325 #endif
326
_spin_unlock(spinlock_t * lock)327 void __lockfunc _spin_unlock(spinlock_t *lock)
328 {
329 spin_release(&lock->dep_map, 1, _RET_IP_);
330 _raw_spin_unlock(lock);
331 preempt_enable();
332 }
333 EXPORT_SYMBOL(_spin_unlock);
334
_write_unlock(rwlock_t * lock)335 void __lockfunc _write_unlock(rwlock_t *lock)
336 {
337 rwlock_release(&lock->dep_map, 1, _RET_IP_);
338 _raw_write_unlock(lock);
339 preempt_enable();
340 }
341 EXPORT_SYMBOL(_write_unlock);
342
_read_unlock(rwlock_t * lock)343 void __lockfunc _read_unlock(rwlock_t *lock)
344 {
345 rwlock_release(&lock->dep_map, 1, _RET_IP_);
346 _raw_read_unlock(lock);
347 preempt_enable();
348 }
349 EXPORT_SYMBOL(_read_unlock);
350
_spin_unlock_irqrestore(spinlock_t * lock,unsigned long flags)351 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
352 {
353 spin_release(&lock->dep_map, 1, _RET_IP_);
354 _raw_spin_unlock(lock);
355 local_irq_restore(flags);
356 preempt_enable();
357 }
358 EXPORT_SYMBOL(_spin_unlock_irqrestore);
359
_spin_unlock_irq(spinlock_t * lock)360 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
361 {
362 spin_release(&lock->dep_map, 1, _RET_IP_);
363 _raw_spin_unlock(lock);
364 local_irq_enable();
365 preempt_enable();
366 }
367 EXPORT_SYMBOL(_spin_unlock_irq);
368
_spin_unlock_bh(spinlock_t * lock)369 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
370 {
371 spin_release(&lock->dep_map, 1, _RET_IP_);
372 _raw_spin_unlock(lock);
373 preempt_enable_no_resched();
374 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
375 }
376 EXPORT_SYMBOL(_spin_unlock_bh);
377
_read_unlock_irqrestore(rwlock_t * lock,unsigned long flags)378 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
379 {
380 rwlock_release(&lock->dep_map, 1, _RET_IP_);
381 _raw_read_unlock(lock);
382 local_irq_restore(flags);
383 preempt_enable();
384 }
385 EXPORT_SYMBOL(_read_unlock_irqrestore);
386
_read_unlock_irq(rwlock_t * lock)387 void __lockfunc _read_unlock_irq(rwlock_t *lock)
388 {
389 rwlock_release(&lock->dep_map, 1, _RET_IP_);
390 _raw_read_unlock(lock);
391 local_irq_enable();
392 preempt_enable();
393 }
394 EXPORT_SYMBOL(_read_unlock_irq);
395
_read_unlock_bh(rwlock_t * lock)396 void __lockfunc _read_unlock_bh(rwlock_t *lock)
397 {
398 rwlock_release(&lock->dep_map, 1, _RET_IP_);
399 _raw_read_unlock(lock);
400 preempt_enable_no_resched();
401 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
402 }
403 EXPORT_SYMBOL(_read_unlock_bh);
404
_write_unlock_irqrestore(rwlock_t * lock,unsigned long flags)405 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
406 {
407 rwlock_release(&lock->dep_map, 1, _RET_IP_);
408 _raw_write_unlock(lock);
409 local_irq_restore(flags);
410 preempt_enable();
411 }
412 EXPORT_SYMBOL(_write_unlock_irqrestore);
413
_write_unlock_irq(rwlock_t * lock)414 void __lockfunc _write_unlock_irq(rwlock_t *lock)
415 {
416 rwlock_release(&lock->dep_map, 1, _RET_IP_);
417 _raw_write_unlock(lock);
418 local_irq_enable();
419 preempt_enable();
420 }
421 EXPORT_SYMBOL(_write_unlock_irq);
422
_write_unlock_bh(rwlock_t * lock)423 void __lockfunc _write_unlock_bh(rwlock_t *lock)
424 {
425 rwlock_release(&lock->dep_map, 1, _RET_IP_);
426 _raw_write_unlock(lock);
427 preempt_enable_no_resched();
428 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
429 }
430 EXPORT_SYMBOL(_write_unlock_bh);
431
_spin_trylock_bh(spinlock_t * lock)432 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
433 {
434 local_bh_disable();
435 preempt_disable();
436 if (_raw_spin_trylock(lock)) {
437 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
438 return 1;
439 }
440
441 preempt_enable_no_resched();
442 local_bh_enable_ip((unsigned long)__builtin_return_address(0));
443 return 0;
444 }
445 EXPORT_SYMBOL(_spin_trylock_bh);
446
in_lock_functions(unsigned long addr)447 notrace int in_lock_functions(unsigned long addr)
448 {
449 /* Linker adds these: start and end of __lockfunc functions */
450 extern char __lock_text_start[], __lock_text_end[];
451
452 return addr >= (unsigned long)__lock_text_start
453 && addr < (unsigned long)__lock_text_end;
454 }
455 EXPORT_SYMBOL(in_lock_functions);
456