1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #include <asm/spr-regs.h> 4 5 #ifdef __ATOMIC_LIB__ 6 7 #ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS 8 9 #define ATOMIC_QUALS 10 #define ATOMIC_EXPORT(x) EXPORT_SYMBOL(x) 11 12 #else /* !OUTOFLINE && LIB */ 13 14 #define ATOMIC_OP_RETURN(op) 15 #define ATOMIC_FETCH_OP(op) 16 17 #endif /* OUTOFLINE */ 18 19 #else /* !__ATOMIC_LIB__ */ 20 21 #define ATOMIC_EXPORT(x) 22 23 #ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS 24 25 #define ATOMIC_OP_RETURN(op) \ 26 extern int __atomic_##op##_return(int i, int *v); \ 27 extern long long __atomic64_##op##_return(long long i, long long *v); 28 29 #define ATOMIC_FETCH_OP(op) \ 30 extern int __atomic32_fetch_##op(int i, int *v); \ 31 extern long long __atomic64_fetch_##op(long long i, long long *v); 32 33 #else /* !OUTOFLINE && !LIB */ 34 35 #define ATOMIC_QUALS static inline 36 37 #endif /* OUTOFLINE */ 38 #endif /* __ATOMIC_LIB__ */ 39 40 41 /* 42 * Note on the 64 bit inline asm variants... 43 * 44 * CSTD is a conditional instruction and needs a constrained memory reference. 45 * Normally 'U' provides the correct constraints for conditional instructions 46 * and this is used for the 32 bit version, however 'U' does not appear to work 47 * for 64 bit values (gcc-4.9) 48 * 49 * The exact constraint is that conditional instructions cannot deal with an 50 * immediate displacement in the memory reference, so what we do is we read the 51 * address through a volatile cast into a local variable in order to insure we 52 * _have_ to compute the correct address without displacement. This allows us 53 * to use the regular 'm' for the memory address. 54 * 55 * Furthermore, the %Ln operand, which prints the low word register (r+1), 56 * really only works for registers, this means we cannot allow immediate values 57 * for the 64 bit versions -- like we do for the 32 bit ones. 58 * 59 */ 60 61 #ifndef ATOMIC_OP_RETURN 62 #define ATOMIC_OP_RETURN(op) \ 63 ATOMIC_QUALS int __atomic_##op##_return(int i, int *v) \ 64 { \ 65 int val; \ 66 \ 67 asm volatile( \ 68 "0: \n" \ 69 " orcc gr0,gr0,gr0,icc3 \n" \ 70 " ckeq icc3,cc7 \n" \ 71 " ld.p %M0,%1 \n" \ 72 " orcr cc7,cc7,cc3 \n" \ 73 " "#op"%I2 %1,%2,%1 \n" \ 74 " cst.p %1,%M0 ,cc3,#1 \n" \ 75 " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ 76 " beq icc3,#0,0b \n" \ 77 : "+U"(*v), "=&r"(val) \ 78 : "NPr"(i) \ 79 : "memory", "cc7", "cc3", "icc3" \ 80 ); \ 81 \ 82 return val; \ 83 } \ 84 ATOMIC_EXPORT(__atomic_##op##_return); \ 85 \ 86 ATOMIC_QUALS long long __atomic64_##op##_return(long long i, long long *v) \ 87 { \ 88 long long *__v = READ_ONCE(v); \ 89 long long val; \ 90 \ 91 asm volatile( \ 92 "0: \n" \ 93 " orcc gr0,gr0,gr0,icc3 \n" \ 94 " ckeq icc3,cc7 \n" \ 95 " ldd.p %M0,%1 \n" \ 96 " orcr cc7,cc7,cc3 \n" \ 97 " "#op"cc %L1,%L2,%L1,icc0 \n" \ 98 " "#op"x %1,%2,%1,icc0 \n" \ 99 " cstd.p %1,%M0 ,cc3,#1 \n" \ 100 " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ 101 " beq icc3,#0,0b \n" \ 102 : "+m"(*__v), "=&e"(val) \ 103 : "e"(i) \ 104 : "memory", "cc7", "cc3", "icc0", "icc3" \ 105 ); \ 106 \ 107 return val; \ 108 } \ 109 ATOMIC_EXPORT(__atomic64_##op##_return); 110 #endif 111 112 #ifndef ATOMIC_FETCH_OP 113 #define ATOMIC_FETCH_OP(op) \ 114 ATOMIC_QUALS int __atomic32_fetch_##op(int i, int *v) \ 115 { \ 116 int old, tmp; \ 117 \ 118 asm volatile( \ 119 "0: \n" \ 120 " orcc gr0,gr0,gr0,icc3 \n" \ 121 " ckeq icc3,cc7 \n" \ 122 " ld.p %M0,%1 \n" \ 123 " orcr cc7,cc7,cc3 \n" \ 124 " "#op"%I3 %1,%3,%2 \n" \ 125 " cst.p %2,%M0 ,cc3,#1 \n" \ 126 " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ 127 " beq icc3,#0,0b \n" \ 128 : "+U"(*v), "=&r"(old), "=r"(tmp) \ 129 : "NPr"(i) \ 130 : "memory", "cc7", "cc3", "icc3" \ 131 ); \ 132 \ 133 return old; \ 134 } \ 135 ATOMIC_EXPORT(__atomic32_fetch_##op); \ 136 \ 137 ATOMIC_QUALS long long __atomic64_fetch_##op(long long i, long long *v) \ 138 { \ 139 long long *__v = READ_ONCE(v); \ 140 long long old, tmp; \ 141 \ 142 asm volatile( \ 143 "0: \n" \ 144 " orcc gr0,gr0,gr0,icc3 \n" \ 145 " ckeq icc3,cc7 \n" \ 146 " ldd.p %M0,%1 \n" \ 147 " orcr cc7,cc7,cc3 \n" \ 148 " "#op" %L1,%L3,%L2 \n" \ 149 " "#op" %1,%3,%2 \n" \ 150 " cstd.p %2,%M0 ,cc3,#1 \n" \ 151 " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ 152 " beq icc3,#0,0b \n" \ 153 : "+m"(*__v), "=&e"(old), "=e"(tmp) \ 154 : "e"(i) \ 155 : "memory", "cc7", "cc3", "icc3" \ 156 ); \ 157 \ 158 return old; \ 159 } \ 160 ATOMIC_EXPORT(__atomic64_fetch_##op); 161 #endif 162 163 ATOMIC_FETCH_OP(or) 164 ATOMIC_FETCH_OP(and) 165 ATOMIC_FETCH_OP(xor) 166 ATOMIC_FETCH_OP(add) 167 ATOMIC_FETCH_OP(sub) 168 169 ATOMIC_OP_RETURN(add) 170 ATOMIC_OP_RETURN(sub) 171 172 #undef ATOMIC_FETCH_OP 173 #undef ATOMIC_OP_RETURN 174 #undef ATOMIC_QUALS 175 #undef ATOMIC_EXPORT 176