1 /*
2 * Out of line spinlock code.
3 *
4 * Copyright IBM Corp. 2004, 2006
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
6 */
7
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/spinlock.h>
11 #include <linux/init.h>
12 #include <linux/smp.h>
13 #include <asm/io.h>
14
15 int spin_retry = 1000;
16
17 /**
18 * spin_retry= parameter
19 */
spin_retry_setup(char * str)20 static int __init spin_retry_setup(char *str)
21 {
22 spin_retry = simple_strtoul(str, &str, 0);
23 return 1;
24 }
25 __setup("spin_retry=", spin_retry_setup);
26
arch_spin_lock_wait(arch_spinlock_t * lp)27 void arch_spin_lock_wait(arch_spinlock_t *lp)
28 {
29 unsigned int cpu = SPINLOCK_LOCKVAL;
30 unsigned int owner;
31 int count;
32
33 while (1) {
34 owner = ACCESS_ONCE(lp->lock);
35 /* Try to get the lock if it is free. */
36 if (!owner) {
37 if (_raw_compare_and_swap(&lp->lock, 0, cpu))
38 return;
39 continue;
40 }
41 /* Check if the lock owner is running. */
42 if (!smp_vcpu_scheduled(~owner)) {
43 smp_yield_cpu(~owner);
44 continue;
45 }
46 /* Loop for a while on the lock value. */
47 count = spin_retry;
48 do {
49 owner = ACCESS_ONCE(lp->lock);
50 } while (owner && count-- > 0);
51 if (!owner)
52 continue;
53 /*
54 * For multiple layers of hypervisors, e.g. z/VM + LPAR
55 * yield the CPU if the lock is still unavailable.
56 */
57 if (!MACHINE_IS_LPAR)
58 smp_yield_cpu(~owner);
59 }
60 }
61 EXPORT_SYMBOL(arch_spin_lock_wait);
62
arch_spin_lock_wait_flags(arch_spinlock_t * lp,unsigned long flags)63 void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
64 {
65 unsigned int cpu = SPINLOCK_LOCKVAL;
66 unsigned int owner;
67 int count;
68
69 local_irq_restore(flags);
70 while (1) {
71 owner = ACCESS_ONCE(lp->lock);
72 /* Try to get the lock if it is free. */
73 if (!owner) {
74 local_irq_disable();
75 if (_raw_compare_and_swap(&lp->lock, 0, cpu))
76 return;
77 local_irq_restore(flags);
78 }
79 /* Check if the lock owner is running. */
80 if (!smp_vcpu_scheduled(~owner)) {
81 smp_yield_cpu(~owner);
82 continue;
83 }
84 /* Loop for a while on the lock value. */
85 count = spin_retry;
86 do {
87 owner = ACCESS_ONCE(lp->lock);
88 } while (owner && count-- > 0);
89 if (!owner)
90 continue;
91 /*
92 * For multiple layers of hypervisors, e.g. z/VM + LPAR
93 * yield the CPU if the lock is still unavailable.
94 */
95 if (!MACHINE_IS_LPAR)
96 smp_yield_cpu(~owner);
97 }
98 }
99 EXPORT_SYMBOL(arch_spin_lock_wait_flags);
100
arch_spin_trylock_retry(arch_spinlock_t * lp)101 int arch_spin_trylock_retry(arch_spinlock_t *lp)
102 {
103 int count;
104
105 for (count = spin_retry; count > 0; count--)
106 if (arch_spin_trylock_once(lp))
107 return 1;
108 return 0;
109 }
110 EXPORT_SYMBOL(arch_spin_trylock_retry);
111
_raw_read_lock_wait(arch_rwlock_t * rw)112 void _raw_read_lock_wait(arch_rwlock_t *rw)
113 {
114 unsigned int owner, old;
115 int count = spin_retry;
116
117 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
118 __RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
119 #endif
120 owner = 0;
121 while (1) {
122 if (count-- <= 0) {
123 if (owner && !smp_vcpu_scheduled(~owner))
124 smp_yield_cpu(~owner);
125 count = spin_retry;
126 }
127 old = ACCESS_ONCE(rw->lock);
128 owner = ACCESS_ONCE(rw->owner);
129 if ((int) old < 0)
130 continue;
131 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
132 return;
133 }
134 }
135 EXPORT_SYMBOL(_raw_read_lock_wait);
136
_raw_read_trylock_retry(arch_rwlock_t * rw)137 int _raw_read_trylock_retry(arch_rwlock_t *rw)
138 {
139 unsigned int old;
140 int count = spin_retry;
141
142 while (count-- > 0) {
143 old = ACCESS_ONCE(rw->lock);
144 if ((int) old < 0)
145 continue;
146 if (_raw_compare_and_swap(&rw->lock, old, old + 1))
147 return 1;
148 }
149 return 0;
150 }
151 EXPORT_SYMBOL(_raw_read_trylock_retry);
152
153 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
154
_raw_write_lock_wait(arch_rwlock_t * rw,unsigned int prev)155 void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
156 {
157 unsigned int owner, old;
158 int count = spin_retry;
159
160 owner = 0;
161 while (1) {
162 if (count-- <= 0) {
163 if (owner && !smp_vcpu_scheduled(~owner))
164 smp_yield_cpu(~owner);
165 count = spin_retry;
166 }
167 old = ACCESS_ONCE(rw->lock);
168 owner = ACCESS_ONCE(rw->owner);
169 smp_rmb();
170 if ((int) old >= 0) {
171 prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
172 old = prev;
173 }
174 if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
175 break;
176 }
177 }
178 EXPORT_SYMBOL(_raw_write_lock_wait);
179
180 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
181
_raw_write_lock_wait(arch_rwlock_t * rw)182 void _raw_write_lock_wait(arch_rwlock_t *rw)
183 {
184 unsigned int owner, old, prev;
185 int count = spin_retry;
186
187 prev = 0x80000000;
188 owner = 0;
189 while (1) {
190 if (count-- <= 0) {
191 if (owner && !smp_vcpu_scheduled(~owner))
192 smp_yield_cpu(~owner);
193 count = spin_retry;
194 }
195 old = ACCESS_ONCE(rw->lock);
196 owner = ACCESS_ONCE(rw->owner);
197 if ((int) old >= 0 &&
198 _raw_compare_and_swap(&rw->lock, old, old | 0x80000000))
199 prev = old;
200 else
201 smp_rmb();
202 if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
203 break;
204 }
205 }
206 EXPORT_SYMBOL(_raw_write_lock_wait);
207
208 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
209
_raw_write_trylock_retry(arch_rwlock_t * rw)210 int _raw_write_trylock_retry(arch_rwlock_t *rw)
211 {
212 unsigned int old;
213 int count = spin_retry;
214
215 while (count-- > 0) {
216 old = ACCESS_ONCE(rw->lock);
217 if (old)
218 continue;
219 if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
220 return 1;
221 }
222 return 0;
223 }
224 EXPORT_SYMBOL(_raw_write_trylock_retry);
225
arch_lock_relax(unsigned int cpu)226 void arch_lock_relax(unsigned int cpu)
227 {
228 if (!cpu)
229 return;
230 if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu))
231 return;
232 smp_yield_cpu(~cpu);
233 }
234 EXPORT_SYMBOL(arch_lock_relax);
235