1 /*
2 * Out of line spinlock code.
3 *
4 * Copyright IBM Corp. 2004, 2006
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
6 */
7
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/spinlock.h>
11 #include <linux/init.h>
12 #include <linux/smp.h>
13 #include <asm/io.h>
14
15 int spin_retry = -1;
16
spin_retry_init(void)17 static int __init spin_retry_init(void)
18 {
19 if (spin_retry < 0)
20 spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
21 return 0;
22 }
23 early_initcall(spin_retry_init);
24
25 /**
26 * spin_retry= parameter
27 */
spin_retry_setup(char * str)28 static int __init spin_retry_setup(char *str)
29 {
30 spin_retry = simple_strtoul(str, &str, 0);
31 return 1;
32 }
33 __setup("spin_retry=", spin_retry_setup);
34
_raw_compare_and_delay(unsigned int * lock,unsigned int old)35 static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
36 {
37 asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
38 }
39
arch_spin_lock_wait(arch_spinlock_t * lp)40 void arch_spin_lock_wait(arch_spinlock_t *lp)
41 {
42 unsigned int cpu = SPINLOCK_LOCKVAL;
43 unsigned int owner;
44 int count;
45
46 while (1) {
47 owner = ACCESS_ONCE(lp->lock);
48 /* Try to get the lock if it is free. */
49 if (!owner) {
50 if (_raw_compare_and_swap(&lp->lock, 0, cpu))
51 return;
52 continue;
53 }
54 /* Check if the lock owner is running. */
55 if (!smp_vcpu_scheduled(~owner)) {
56 smp_yield_cpu(~owner);
57 continue;
58 }
59 /* Loop for a while on the lock value. */
60 count = spin_retry;
61 do {
62 if (MACHINE_HAS_CAD)
63 _raw_compare_and_delay(&lp->lock, owner);
64 owner = ACCESS_ONCE(lp->lock);
65 } while (owner && count-- > 0);
66 if (!owner)
67 continue;
68 /*
69 * For multiple layers of hypervisors, e.g. z/VM + LPAR
70 * yield the CPU if the lock is still unavailable.
71 */
72 if (!MACHINE_IS_LPAR)
73 smp_yield_cpu(~owner);
74 }
75 }
76 EXPORT_SYMBOL(arch_spin_lock_wait);
77
arch_spin_lock_wait_flags(arch_spinlock_t * lp,unsigned long flags)78 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
79 {
80 unsigned int cpu = SPINLOCK_LOCKVAL;
81 unsigned int owner;
82 int count;
83
84 local_irq_restore(flags);
85 while (1) {
86 owner = ACCESS_ONCE(lp->lock);
87 /* Try to get the lock if it is free. */
88 if (!owner) {
89 local_irq_disable();
90 if (_raw_compare_and_swap(&lp->lock, 0, cpu))
91 return;
92 local_irq_restore(flags);
93 }
94 /* Check if the lock owner is running. */
95 if (!smp_vcpu_scheduled(~owner)) {
96 smp_yield_cpu(~owner);
97 continue;
98 }
99 /* Loop for a while on the lock value. */
100 count = spin_retry;
101 do {
102 if (MACHINE_HAS_CAD)
103 _raw_compare_and_delay(&lp->lock, owner);
104 owner = ACCESS_ONCE(lp->lock);
105 } while (owner && count-- > 0);
106 if (!owner)
107 continue;
108 /*
109 * For multiple layers of hypervisors, e.g. z/VM + LPAR
110 * yield the CPU if the lock is still unavailable.
111 */
112 if (!MACHINE_IS_LPAR)
113 smp_yield_cpu(~owner);
114 }
115 }
116 EXPORT_SYMBOL(arch_spin_lock_wait_flags);
117
arch_spin_trylock_retry(arch_spinlock_t * lp)118 int arch_spin_trylock_retry(arch_spinlock_t *lp)
119 {
120 unsigned int cpu = SPINLOCK_LOCKVAL;
121 unsigned int owner;
122 int count;
123
124 for (count = spin_retry; count > 0; count--) {
125 owner = ACCESS_ONCE(lp->lock);
126 /* Try to get the lock if it is free. */
127 if (!owner) {
128 if (_raw_compare_and_swap(&lp->lock, 0, cpu))
129 return 1;
130 } else if (MACHINE_HAS_CAD)
131 _raw_compare_and_delay(&lp->lock, owner);
132 }
133 return 0;
134 }
135 EXPORT_SYMBOL(arch_spin_trylock_retry);
136
_raw_read_lock_wait(arch_rwlock_t * rw)137 void _raw_read_lock_wait(arch_rwlock_t *rw)
138 {
139 unsigned int owner, old;
140 int count = spin_retry;
141
142 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
143 __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
144 #endif
145 owner = 0;
146 while (1) {
147 if (count-- <= 0) {
148 if (owner && !smp_vcpu_scheduled(~owner))
149 smp_yield_cpu(~owner);
150 count = spin_retry;
151 }
152 old = ACCESS_ONCE(rw->lock);
153 owner = ACCESS_ONCE(rw->owner);
154 if ((int) old < 0) {
155 if (MACHINE_HAS_CAD)
156 _raw_compare_and_delay(&rw->lock, old);
157 continue;
158 }
159 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
160 return;
161 }
162 }
163 EXPORT_SYMBOL(_raw_read_lock_wait);
164
_raw_read_trylock_retry(arch_rwlock_t * rw)165 int _raw_read_trylock_retry(arch_rwlock_t *rw)
166 {
167 unsigned int old;
168 int count = spin_retry;
169
170 while (count-- > 0) {
171 old = ACCESS_ONCE(rw->lock);
172 if ((int) old < 0) {
173 if (MACHINE_HAS_CAD)
174 _raw_compare_and_delay(&rw->lock, old);
175 continue;
176 }
177 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
178 return 1;
179 }
180 return 0;
181 }
182 EXPORT_SYMBOL(_raw_read_trylock_retry);
183
184 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
185
_raw_write_lock_wait(arch_rwlock_t * rw,unsigned int prev)186 void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
187 {
188 unsigned int owner, old;
189 int count = spin_retry;
190
191 owner = 0;
192 while (1) {
193 if (count-- <= 0) {
194 if (owner && !smp_vcpu_scheduled(~owner))
195 smp_yield_cpu(~owner);
196 count = spin_retry;
197 }
198 old = ACCESS_ONCE(rw->lock);
199 owner = ACCESS_ONCE(rw->owner);
200 smp_mb();
201 if ((int) old >= 0) {
202 prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
203 old = prev;
204 }
205 if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
206 break;
207 if (MACHINE_HAS_CAD)
208 _raw_compare_and_delay(&rw->lock, old);
209 }
210 }
211 EXPORT_SYMBOL(_raw_write_lock_wait);
212
213 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
214
_raw_write_lock_wait(arch_rwlock_t * rw)215 void _raw_write_lock_wait(arch_rwlock_t *rw)
216 {
217 unsigned int owner, old, prev;
218 int count = spin_retry;
219
220 prev = 0x80000000;
221 owner = 0;
222 while (1) {
223 if (count-- <= 0) {
224 if (owner && !smp_vcpu_scheduled(~owner))
225 smp_yield_cpu(~owner);
226 count = spin_retry;
227 }
228 old = ACCESS_ONCE(rw->lock);
229 owner = ACCESS_ONCE(rw->owner);
230 if ((int) old >= 0 &&
231 _raw_compare_and_swap(&rw->lock, old, old | 0x80000000))
232 prev = old;
233 else
234 smp_mb();
235 if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
236 break;
237 if (MACHINE_HAS_CAD)
238 _raw_compare_and_delay(&rw->lock, old);
239 }
240 }
241 EXPORT_SYMBOL(_raw_write_lock_wait);
242
243 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
244
_raw_write_trylock_retry(arch_rwlock_t * rw)245 int _raw_write_trylock_retry(arch_rwlock_t *rw)
246 {
247 unsigned int old;
248 int count = spin_retry;
249
250 while (count-- > 0) {
251 old = ACCESS_ONCE(rw->lock);
252 if (old) {
253 if (MACHINE_HAS_CAD)
254 _raw_compare_and_delay(&rw->lock, old);
255 continue;
256 }
257 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
258 return 1;
259 }
260 return 0;
261 }
262 EXPORT_SYMBOL(_raw_write_trylock_retry);
263
arch_lock_relax(unsigned int cpu)264 void arch_lock_relax(unsigned int cpu)
265 {
266 if (!cpu)
267 return;
268 if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu))
269 return;
270 smp_yield_cpu(~cpu);
271 }
272 EXPORT_SYMBOL(arch_lock_relax);
273