• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef __ARCH_S390_ATOMIC__
2 #define __ARCH_S390_ATOMIC__
3 
4 /*
5  * Copyright 1999,2009 IBM Corp.
6  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
7  *	      Denis Joseph Barrow,
8  *	      Arnd Bergmann <arndb@de.ibm.com>,
9  *
10  * Atomic operations that C can't guarantee us.
11  * Useful for resource counting etc.
12  * s390 uses 'Compare And Swap' for atomicity in SMP environment.
13  *
14  */
15 
16 #include <linux/compiler.h>
17 #include <linux/types.h>
18 #include <asm/cmpxchg.h>
19 
20 #define ATOMIC_INIT(i)  { (i) }
21 
22 #define __CS_LOOP(ptr, op_val, op_string) ({				\
23 	int old_val, new_val;						\
24 	asm volatile(							\
25 		"	l	%0,%2\n"				\
26 		"0:	lr	%1,%0\n"				\
27 		op_string "	%1,%3\n"				\
28 		"	cs	%0,%1,%2\n"				\
29 		"	jl	0b"					\
30 		: "=&d" (old_val), "=&d" (new_val),			\
31 		  "=Q" (((atomic_t *)(ptr))->counter)			\
32 		: "d" (op_val),	 "Q" (((atomic_t *)(ptr))->counter)	\
33 		: "cc", "memory");					\
34 	new_val;							\
35 })
36 
atomic_read(const atomic_t * v)37 static inline int atomic_read(const atomic_t *v)
38 {
39 	int c;
40 
41 	asm volatile(
42 		"	l	%0,%1\n"
43 		: "=d" (c) : "Q" (v->counter));
44 	return c;
45 }
46 
atomic_set(atomic_t * v,int i)47 static inline void atomic_set(atomic_t *v, int i)
48 {
49 	asm volatile(
50 		"	st	%1,%0\n"
51 		: "=Q" (v->counter) : "d" (i));
52 }
53 
atomic_add_return(int i,atomic_t * v)54 static inline int atomic_add_return(int i, atomic_t *v)
55 {
56 	return __CS_LOOP(v, i, "ar");
57 }
58 #define atomic_add(_i, _v)		atomic_add_return(_i, _v)
59 #define atomic_add_negative(_i, _v)	(atomic_add_return(_i, _v) < 0)
60 #define atomic_inc(_v)			atomic_add_return(1, _v)
61 #define atomic_inc_return(_v)		atomic_add_return(1, _v)
62 #define atomic_inc_and_test(_v)		(atomic_add_return(1, _v) == 0)
63 
atomic_sub_return(int i,atomic_t * v)64 static inline int atomic_sub_return(int i, atomic_t *v)
65 {
66 	return __CS_LOOP(v, i, "sr");
67 }
68 #define atomic_sub(_i, _v)		atomic_sub_return(_i, _v)
69 #define atomic_sub_and_test(_i, _v)	(atomic_sub_return(_i, _v) == 0)
70 #define atomic_dec(_v)			atomic_sub_return(1, _v)
71 #define atomic_dec_return(_v)		atomic_sub_return(1, _v)
72 #define atomic_dec_and_test(_v)		(atomic_sub_return(1, _v) == 0)
73 
atomic_clear_mask(unsigned long mask,atomic_t * v)74 static inline void atomic_clear_mask(unsigned long mask, atomic_t *v)
75 {
76 	__CS_LOOP(v, ~mask, "nr");
77 }
78 
atomic_set_mask(unsigned long mask,atomic_t * v)79 static inline void atomic_set_mask(unsigned long mask, atomic_t *v)
80 {
81 	__CS_LOOP(v, mask, "or");
82 }
83 
84 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
85 
atomic_cmpxchg(atomic_t * v,int old,int new)86 static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
87 {
88 	asm volatile(
89 		"	cs	%0,%2,%1"
90 		: "+d" (old), "=Q" (v->counter)
91 		: "d" (new), "Q" (v->counter)
92 		: "cc", "memory");
93 	return old;
94 }
95 
__atomic_add_unless(atomic_t * v,int a,int u)96 static inline int __atomic_add_unless(atomic_t *v, int a, int u)
97 {
98 	int c, old;
99 	c = atomic_read(v);
100 	for (;;) {
101 		if (unlikely(c == u))
102 			break;
103 		old = atomic_cmpxchg(v, c, c + a);
104 		if (likely(old == c))
105 			break;
106 		c = old;
107 	}
108 	return c;
109 }
110 
111 
112 #undef __CS_LOOP
113 
114 #define ATOMIC64_INIT(i)  { (i) }
115 
116 #ifdef CONFIG_64BIT
117 
118 #define __CSG_LOOP(ptr, op_val, op_string) ({				\
119 	long long old_val, new_val;					\
120 	asm volatile(							\
121 		"	lg	%0,%2\n"				\
122 		"0:	lgr	%1,%0\n"				\
123 		op_string "	%1,%3\n"				\
124 		"	csg	%0,%1,%2\n"				\
125 		"	jl	0b"					\
126 		: "=&d" (old_val), "=&d" (new_val),			\
127 		  "=Q" (((atomic_t *)(ptr))->counter)			\
128 		: "d" (op_val),	"Q" (((atomic_t *)(ptr))->counter)	\
129 		: "cc", "memory");					\
130 	new_val;							\
131 })
132 
atomic64_read(const atomic64_t * v)133 static inline long long atomic64_read(const atomic64_t *v)
134 {
135 	long long c;
136 
137 	asm volatile(
138 		"	lg	%0,%1\n"
139 		: "=d" (c) : "Q" (v->counter));
140 	return c;
141 }
142 
atomic64_set(atomic64_t * v,long long i)143 static inline void atomic64_set(atomic64_t *v, long long i)
144 {
145 	asm volatile(
146 		"	stg	%1,%0\n"
147 		: "=Q" (v->counter) : "d" (i));
148 }
149 
atomic64_add_return(long long i,atomic64_t * v)150 static inline long long atomic64_add_return(long long i, atomic64_t *v)
151 {
152 	return __CSG_LOOP(v, i, "agr");
153 }
154 
atomic64_sub_return(long long i,atomic64_t * v)155 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
156 {
157 	return __CSG_LOOP(v, i, "sgr");
158 }
159 
atomic64_clear_mask(unsigned long mask,atomic64_t * v)160 static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
161 {
162 	__CSG_LOOP(v, ~mask, "ngr");
163 }
164 
atomic64_set_mask(unsigned long mask,atomic64_t * v)165 static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
166 {
167 	__CSG_LOOP(v, mask, "ogr");
168 }
169 
170 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
171 
atomic64_cmpxchg(atomic64_t * v,long long old,long long new)172 static inline long long atomic64_cmpxchg(atomic64_t *v,
173 					     long long old, long long new)
174 {
175 	asm volatile(
176 		"	csg	%0,%2,%1"
177 		: "+d" (old), "=Q" (v->counter)
178 		: "d" (new), "Q" (v->counter)
179 		: "cc", "memory");
180 	return old;
181 }
182 
183 #undef __CSG_LOOP
184 
185 #else /* CONFIG_64BIT */
186 
187 typedef struct {
188 	long long counter;
189 } atomic64_t;
190 
atomic64_read(const atomic64_t * v)191 static inline long long atomic64_read(const atomic64_t *v)
192 {
193 	register_pair rp;
194 
195 	asm volatile(
196 		"	lm	%0,%N0,%1"
197 		: "=&d" (rp) : "Q" (v->counter)	);
198 	return rp.pair;
199 }
200 
atomic64_set(atomic64_t * v,long long i)201 static inline void atomic64_set(atomic64_t *v, long long i)
202 {
203 	register_pair rp = {.pair = i};
204 
205 	asm volatile(
206 		"	stm	%1,%N1,%0"
207 		: "=Q" (v->counter) : "d" (rp) );
208 }
209 
atomic64_xchg(atomic64_t * v,long long new)210 static inline long long atomic64_xchg(atomic64_t *v, long long new)
211 {
212 	register_pair rp_new = {.pair = new};
213 	register_pair rp_old;
214 
215 	asm volatile(
216 		"	lm	%0,%N0,%1\n"
217 		"0:	cds	%0,%2,%1\n"
218 		"	jl	0b\n"
219 		: "=&d" (rp_old), "=Q" (v->counter)
220 		: "d" (rp_new), "Q" (v->counter)
221 		: "cc");
222 	return rp_old.pair;
223 }
224 
atomic64_cmpxchg(atomic64_t * v,long long old,long long new)225 static inline long long atomic64_cmpxchg(atomic64_t *v,
226 					 long long old, long long new)
227 {
228 	register_pair rp_old = {.pair = old};
229 	register_pair rp_new = {.pair = new};
230 
231 	asm volatile(
232 		"	cds	%0,%2,%1"
233 		: "+&d" (rp_old), "=Q" (v->counter)
234 		: "d" (rp_new), "Q" (v->counter)
235 		: "cc");
236 	return rp_old.pair;
237 }
238 
239 
atomic64_add_return(long long i,atomic64_t * v)240 static inline long long atomic64_add_return(long long i, atomic64_t *v)
241 {
242 	long long old, new;
243 
244 	do {
245 		old = atomic64_read(v);
246 		new = old + i;
247 	} while (atomic64_cmpxchg(v, old, new) != old);
248 	return new;
249 }
250 
atomic64_sub_return(long long i,atomic64_t * v)251 static inline long long atomic64_sub_return(long long i, atomic64_t *v)
252 {
253 	long long old, new;
254 
255 	do {
256 		old = atomic64_read(v);
257 		new = old - i;
258 	} while (atomic64_cmpxchg(v, old, new) != old);
259 	return new;
260 }
261 
atomic64_set_mask(unsigned long long mask,atomic64_t * v)262 static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v)
263 {
264 	long long old, new;
265 
266 	do {
267 		old = atomic64_read(v);
268 		new = old | mask;
269 	} while (atomic64_cmpxchg(v, old, new) != old);
270 }
271 
atomic64_clear_mask(unsigned long long mask,atomic64_t * v)272 static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
273 {
274 	long long old, new;
275 
276 	do {
277 		old = atomic64_read(v);
278 		new = old & mask;
279 	} while (atomic64_cmpxchg(v, old, new) != old);
280 }
281 
282 #endif /* CONFIG_64BIT */
283 
atomic64_add_unless(atomic64_t * v,long long a,long long u)284 static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
285 {
286 	long long c, old;
287 
288 	c = atomic64_read(v);
289 	for (;;) {
290 		if (unlikely(c == u))
291 			break;
292 		old = atomic64_cmpxchg(v, c, c + a);
293 		if (likely(old == c))
294 			break;
295 		c = old;
296 	}
297 	return c != u;
298 }
299 
atomic64_dec_if_positive(atomic64_t * v)300 static inline long long atomic64_dec_if_positive(atomic64_t *v)
301 {
302 	long long c, old, dec;
303 
304 	c = atomic64_read(v);
305 	for (;;) {
306 		dec = c - 1;
307 		if (unlikely(dec < 0))
308 			break;
309 		old = atomic64_cmpxchg((v), c, dec);
310 		if (likely(old == c))
311 			break;
312 		c = old;
313 	}
314 	return dec;
315 }
316 
317 #define atomic64_add(_i, _v)		atomic64_add_return(_i, _v)
318 #define atomic64_add_negative(_i, _v)	(atomic64_add_return(_i, _v) < 0)
319 #define atomic64_inc(_v)		atomic64_add_return(1, _v)
320 #define atomic64_inc_return(_v)		atomic64_add_return(1, _v)
321 #define atomic64_inc_and_test(_v)	(atomic64_add_return(1, _v) == 0)
322 #define atomic64_sub(_i, _v)		atomic64_sub_return(_i, _v)
323 #define atomic64_sub_and_test(_i, _v)	(atomic64_sub_return(_i, _v) == 0)
324 #define atomic64_dec(_v)		atomic64_sub_return(1, _v)
325 #define atomic64_dec_return(_v)		atomic64_sub_return(1, _v)
326 #define atomic64_dec_and_test(_v)	(atomic64_sub_return(1, _v) == 0)
327 #define atomic64_inc_not_zero(v)	atomic64_add_unless((v), 1, 0)
328 
329 #define smp_mb__before_atomic_dec()	smp_mb()
330 #define smp_mb__after_atomic_dec()	smp_mb()
331 #define smp_mb__before_atomic_inc()	smp_mb()
332 #define smp_mb__after_atomic_inc()	smp_mb()
333 
334 #endif /* __ARCH_S390_ATOMIC__  */
335