1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * csum_partial_copy - do IP checksumming and copy
4 *
5 * (C) Copyright 1996 Linus Torvalds
6 * accelerated versions (and 21264 assembly versions ) contributed by
7 * Rick Gorton <rick.gorton@alpha-processor.com>
8 *
9 * Don't look at this too closely - you'll go mad. The things
10 * we do for performance..
11 */
12
13 #include <linux/types.h>
14 #include <linux/string.h>
15 #include <linux/uaccess.h>
16
17
18 #define ldq_u(x,y) \
19 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
20
21 #define stq_u(x,y) \
22 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
23
24 #define extql(x,y,z) \
25 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
26
27 #define extqh(x,y,z) \
28 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
29
30 #define mskql(x,y,z) \
31 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
32
33 #define mskqh(x,y,z) \
34 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
35
36 #define insql(x,y,z) \
37 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
38
39 #define insqh(x,y,z) \
40 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
41
42 #define __get_word(insn,x,ptr) \
43 ({ \
44 long __guu_err; \
45 __asm__ __volatile__( \
46 "1: "#insn" %0,%2\n" \
47 "2:\n" \
48 EXC(1b,2b,%0,%1) \
49 : "=r"(x), "=r"(__guu_err) \
50 : "m"(__m(ptr)), "1"(0)); \
51 __guu_err; \
52 })
53
from64to16(unsigned long x)54 static inline unsigned short from64to16(unsigned long x)
55 {
56 /* Using extract instructions is a bit more efficient
57 than the original shift/bitmask version. */
58
59 union {
60 unsigned long ul;
61 unsigned int ui[2];
62 unsigned short us[4];
63 } in_v, tmp_v, out_v;
64
65 in_v.ul = x;
66 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
67
68 /* Since the bits of tmp_v.sh[3] are going to always be zero,
69 we don't have to bother to add that in. */
70 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
71 + (unsigned long) tmp_v.us[2];
72
73 /* Similarly, out_v.us[2] is always zero for the final add. */
74 return out_v.us[0] + out_v.us[1];
75 }
76
77
78
79 /*
80 * Ok. This isn't fun, but this is the EASY case.
81 */
82 static inline unsigned long
csum_partial_cfu_aligned(const unsigned long __user * src,unsigned long * dst,long len)83 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
84 long len)
85 {
86 unsigned long checksum = ~0U;
87 unsigned long carry = 0;
88
89 while (len >= 0) {
90 unsigned long word;
91 if (__get_word(ldq, word, src))
92 return 0;
93 checksum += carry;
94 src++;
95 checksum += word;
96 len -= 8;
97 carry = checksum < word;
98 *dst = word;
99 dst++;
100 }
101 len += 8;
102 checksum += carry;
103 if (len) {
104 unsigned long word, tmp;
105 if (__get_word(ldq, word, src))
106 return 0;
107 tmp = *dst;
108 mskql(word, len, word);
109 checksum += word;
110 mskqh(tmp, len, tmp);
111 carry = checksum < word;
112 *dst = word | tmp;
113 checksum += carry;
114 }
115 return checksum;
116 }
117
118 /*
119 * This is even less fun, but this is still reasonably
120 * easy.
121 */
122 static inline unsigned long
csum_partial_cfu_dest_aligned(const unsigned long __user * src,unsigned long * dst,unsigned long soff,long len)123 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
124 unsigned long *dst,
125 unsigned long soff,
126 long len)
127 {
128 unsigned long first;
129 unsigned long word, carry;
130 unsigned long lastsrc = 7+len+(unsigned long)src;
131 unsigned long checksum = ~0U;
132
133 if (__get_word(ldq_u, first,src))
134 return 0;
135 carry = 0;
136 while (len >= 0) {
137 unsigned long second;
138
139 if (__get_word(ldq_u, second, src+1))
140 return 0;
141 extql(first, soff, word);
142 len -= 8;
143 src++;
144 extqh(second, soff, first);
145 checksum += carry;
146 word |= first;
147 first = second;
148 checksum += word;
149 *dst = word;
150 dst++;
151 carry = checksum < word;
152 }
153 len += 8;
154 checksum += carry;
155 if (len) {
156 unsigned long tmp;
157 unsigned long second;
158 if (__get_word(ldq_u, second, lastsrc))
159 return 0;
160 tmp = *dst;
161 extql(first, soff, word);
162 extqh(second, soff, first);
163 word |= first;
164 mskql(word, len, word);
165 checksum += word;
166 mskqh(tmp, len, tmp);
167 carry = checksum < word;
168 *dst = word | tmp;
169 checksum += carry;
170 }
171 return checksum;
172 }
173
174 /*
175 * This is slightly less fun than the above..
176 */
177 static inline unsigned long
csum_partial_cfu_src_aligned(const unsigned long __user * src,unsigned long * dst,unsigned long doff,long len,unsigned long partial_dest)178 csum_partial_cfu_src_aligned(const unsigned long __user *src,
179 unsigned long *dst,
180 unsigned long doff,
181 long len,
182 unsigned long partial_dest)
183 {
184 unsigned long carry = 0;
185 unsigned long word;
186 unsigned long second_dest;
187 unsigned long checksum = ~0U;
188
189 mskql(partial_dest, doff, partial_dest);
190 while (len >= 0) {
191 if (__get_word(ldq, word, src))
192 return 0;
193 len -= 8;
194 insql(word, doff, second_dest);
195 checksum += carry;
196 stq_u(partial_dest | second_dest, dst);
197 src++;
198 checksum += word;
199 insqh(word, doff, partial_dest);
200 carry = checksum < word;
201 dst++;
202 }
203 len += 8;
204 if (len) {
205 checksum += carry;
206 if (__get_word(ldq, word, src))
207 return 0;
208 mskql(word, len, word);
209 len -= 8;
210 checksum += word;
211 insql(word, doff, second_dest);
212 len += doff;
213 carry = checksum < word;
214 partial_dest |= second_dest;
215 if (len >= 0) {
216 stq_u(partial_dest, dst);
217 if (!len) goto out;
218 dst++;
219 insqh(word, doff, partial_dest);
220 }
221 doff = len;
222 }
223 ldq_u(second_dest, dst);
224 mskqh(second_dest, doff, second_dest);
225 stq_u(partial_dest | second_dest, dst);
226 out:
227 checksum += carry;
228 return checksum;
229 }
230
231 /*
232 * This is so totally un-fun that it's frightening. Don't
233 * look at this too closely, you'll go blind.
234 */
235 static inline unsigned long
csum_partial_cfu_unaligned(const unsigned long __user * src,unsigned long * dst,unsigned long soff,unsigned long doff,long len,unsigned long partial_dest)236 csum_partial_cfu_unaligned(const unsigned long __user * src,
237 unsigned long * dst,
238 unsigned long soff, unsigned long doff,
239 long len, unsigned long partial_dest)
240 {
241 unsigned long carry = 0;
242 unsigned long first;
243 unsigned long lastsrc;
244 unsigned long checksum = ~0U;
245
246 if (__get_word(ldq_u, first, src))
247 return 0;
248 lastsrc = 7+len+(unsigned long)src;
249 mskql(partial_dest, doff, partial_dest);
250 while (len >= 0) {
251 unsigned long second, word;
252 unsigned long second_dest;
253
254 if (__get_word(ldq_u, second, src+1))
255 return 0;
256 extql(first, soff, word);
257 checksum += carry;
258 len -= 8;
259 extqh(second, soff, first);
260 src++;
261 word |= first;
262 first = second;
263 insql(word, doff, second_dest);
264 checksum += word;
265 stq_u(partial_dest | second_dest, dst);
266 carry = checksum < word;
267 insqh(word, doff, partial_dest);
268 dst++;
269 }
270 len += doff;
271 checksum += carry;
272 if (len >= 0) {
273 unsigned long second, word;
274 unsigned long second_dest;
275
276 if (__get_word(ldq_u, second, lastsrc))
277 return 0;
278 extql(first, soff, word);
279 extqh(second, soff, first);
280 word |= first;
281 first = second;
282 mskql(word, len-doff, word);
283 checksum += word;
284 insql(word, doff, second_dest);
285 carry = checksum < word;
286 stq_u(partial_dest | second_dest, dst);
287 if (len) {
288 ldq_u(second_dest, dst+1);
289 insqh(word, doff, partial_dest);
290 mskqh(second_dest, len, second_dest);
291 stq_u(partial_dest | second_dest, dst+1);
292 }
293 checksum += carry;
294 } else {
295 unsigned long second, word;
296 unsigned long second_dest;
297
298 if (__get_word(ldq_u, second, lastsrc))
299 return 0;
300 extql(first, soff, word);
301 extqh(second, soff, first);
302 word |= first;
303 ldq_u(second_dest, dst);
304 mskql(word, len-doff, word);
305 checksum += word;
306 mskqh(second_dest, len, second_dest);
307 carry = checksum < word;
308 insql(word, doff, word);
309 stq_u(partial_dest | word | second_dest, dst);
310 checksum += carry;
311 }
312 return checksum;
313 }
314
__csum_and_copy(const void __user * src,void * dst,int len)315 static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
316 {
317 unsigned long soff = 7 & (unsigned long) src;
318 unsigned long doff = 7 & (unsigned long) dst;
319 unsigned long checksum;
320
321 if (!doff) {
322 if (!soff)
323 checksum = csum_partial_cfu_aligned(
324 (const unsigned long __user *) src,
325 (unsigned long *) dst, len-8);
326 else
327 checksum = csum_partial_cfu_dest_aligned(
328 (const unsigned long __user *) src,
329 (unsigned long *) dst,
330 soff, len-8);
331 } else {
332 unsigned long partial_dest;
333 ldq_u(partial_dest, dst);
334 if (!soff)
335 checksum = csum_partial_cfu_src_aligned(
336 (const unsigned long __user *) src,
337 (unsigned long *) dst,
338 doff, len-8, partial_dest);
339 else
340 checksum = csum_partial_cfu_unaligned(
341 (const unsigned long __user *) src,
342 (unsigned long *) dst,
343 soff, doff, len-8, partial_dest);
344 }
345 return (__force __wsum)from64to16 (checksum);
346 }
347
348 __wsum
csum_and_copy_from_user(const void __user * src,void * dst,int len)349 csum_and_copy_from_user(const void __user *src, void *dst, int len)
350 {
351 if (!access_ok(src, len))
352 return 0;
353 return __csum_and_copy(src, dst, len);
354 }
355 EXPORT_SYMBOL(csum_and_copy_from_user);
356
357 __wsum
csum_partial_copy_nocheck(const void * src,void * dst,int len)358 csum_partial_copy_nocheck(const void *src, void *dst, int len)
359 {
360 return __csum_and_copy((__force const void __user *)src,
361 dst, len);
362 }
363 EXPORT_SYMBOL(csum_partial_copy_nocheck);
364