1 #include <linux/export.h>
2 #include <linux/uio.h>
3 #include <linux/pagemap.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
6 #include <net/checksum.h>
7
8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
9 size_t left; \
10 size_t wanted = n; \
11 __p = i->iov; \
12 __v.iov_len = min(n, __p->iov_len - skip); \
13 if (likely(__v.iov_len)) { \
14 __v.iov_base = __p->iov_base + skip; \
15 left = (STEP); \
16 __v.iov_len -= left; \
17 skip += __v.iov_len; \
18 n -= __v.iov_len; \
19 } else { \
20 left = 0; \
21 } \
22 while (unlikely(!left && n)) { \
23 __p++; \
24 __v.iov_len = min(n, __p->iov_len); \
25 if (unlikely(!__v.iov_len)) \
26 continue; \
27 __v.iov_base = __p->iov_base; \
28 left = (STEP); \
29 __v.iov_len -= left; \
30 skip = __v.iov_len; \
31 n -= __v.iov_len; \
32 } \
33 n = wanted - n; \
34 }
35
36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
37 size_t wanted = n; \
38 __p = i->kvec; \
39 __v.iov_len = min(n, __p->iov_len - skip); \
40 if (likely(__v.iov_len)) { \
41 __v.iov_base = __p->iov_base + skip; \
42 (void)(STEP); \
43 skip += __v.iov_len; \
44 n -= __v.iov_len; \
45 } \
46 while (unlikely(n)) { \
47 __p++; \
48 __v.iov_len = min(n, __p->iov_len); \
49 if (unlikely(!__v.iov_len)) \
50 continue; \
51 __v.iov_base = __p->iov_base; \
52 (void)(STEP); \
53 skip = __v.iov_len; \
54 n -= __v.iov_len; \
55 } \
56 n = wanted; \
57 }
58
59 #define iterate_bvec(i, n, __v, __p, skip, STEP) { \
60 size_t wanted = n; \
61 __p = i->bvec; \
62 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
63 if (likely(__v.bv_len)) { \
64 __v.bv_page = __p->bv_page; \
65 __v.bv_offset = __p->bv_offset + skip; \
66 (void)(STEP); \
67 skip += __v.bv_len; \
68 n -= __v.bv_len; \
69 } \
70 while (unlikely(n)) { \
71 __p++; \
72 __v.bv_len = min_t(size_t, n, __p->bv_len); \
73 if (unlikely(!__v.bv_len)) \
74 continue; \
75 __v.bv_page = __p->bv_page; \
76 __v.bv_offset = __p->bv_offset; \
77 (void)(STEP); \
78 skip = __v.bv_len; \
79 n -= __v.bv_len; \
80 } \
81 n = wanted; \
82 }
83
84 #define iterate_all_kinds(i, n, v, I, B, K) { \
85 size_t skip = i->iov_offset; \
86 if (unlikely(i->type & ITER_BVEC)) { \
87 const struct bio_vec *bvec; \
88 struct bio_vec v; \
89 iterate_bvec(i, n, v, bvec, skip, (B)) \
90 } else if (unlikely(i->type & ITER_KVEC)) { \
91 const struct kvec *kvec; \
92 struct kvec v; \
93 iterate_kvec(i, n, v, kvec, skip, (K)) \
94 } else { \
95 const struct iovec *iov; \
96 struct iovec v; \
97 iterate_iovec(i, n, v, iov, skip, (I)) \
98 } \
99 }
100
101 #define iterate_and_advance(i, n, v, I, B, K) { \
102 size_t skip = i->iov_offset; \
103 if (unlikely(i->type & ITER_BVEC)) { \
104 const struct bio_vec *bvec; \
105 struct bio_vec v; \
106 iterate_bvec(i, n, v, bvec, skip, (B)) \
107 if (skip == bvec->bv_len) { \
108 bvec++; \
109 skip = 0; \
110 } \
111 i->nr_segs -= bvec - i->bvec; \
112 i->bvec = bvec; \
113 } else if (unlikely(i->type & ITER_KVEC)) { \
114 const struct kvec *kvec; \
115 struct kvec v; \
116 iterate_kvec(i, n, v, kvec, skip, (K)) \
117 if (skip == kvec->iov_len) { \
118 kvec++; \
119 skip = 0; \
120 } \
121 i->nr_segs -= kvec - i->kvec; \
122 i->kvec = kvec; \
123 } else { \
124 const struct iovec *iov; \
125 struct iovec v; \
126 iterate_iovec(i, n, v, iov, skip, (I)) \
127 if (skip == iov->iov_len) { \
128 iov++; \
129 skip = 0; \
130 } \
131 i->nr_segs -= iov - i->iov; \
132 i->iov = iov; \
133 } \
134 i->count -= n; \
135 i->iov_offset = skip; \
136 }
137
copy_page_to_iter_iovec(struct page * page,size_t offset,size_t bytes,struct iov_iter * i)138 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
139 struct iov_iter *i)
140 {
141 size_t skip, copy, left, wanted;
142 const struct iovec *iov;
143 char __user *buf;
144 void *kaddr, *from;
145
146 if (unlikely(bytes > i->count))
147 bytes = i->count;
148
149 if (unlikely(!bytes))
150 return 0;
151
152 wanted = bytes;
153 iov = i->iov;
154 skip = i->iov_offset;
155 buf = iov->iov_base + skip;
156 copy = min(bytes, iov->iov_len - skip);
157
158 if (!fault_in_pages_writeable(buf, copy)) {
159 kaddr = kmap_atomic(page);
160 from = kaddr + offset;
161
162 /* first chunk, usually the only one */
163 left = __copy_to_user_inatomic(buf, from, copy);
164 copy -= left;
165 skip += copy;
166 from += copy;
167 bytes -= copy;
168
169 while (unlikely(!left && bytes)) {
170 iov++;
171 buf = iov->iov_base;
172 copy = min(bytes, iov->iov_len);
173 left = __copy_to_user_inatomic(buf, from, copy);
174 copy -= left;
175 skip = copy;
176 from += copy;
177 bytes -= copy;
178 }
179 if (likely(!bytes)) {
180 kunmap_atomic(kaddr);
181 goto done;
182 }
183 offset = from - kaddr;
184 buf += copy;
185 kunmap_atomic(kaddr);
186 copy = min(bytes, iov->iov_len - skip);
187 }
188 /* Too bad - revert to non-atomic kmap */
189 kaddr = kmap(page);
190 from = kaddr + offset;
191 left = __copy_to_user(buf, from, copy);
192 copy -= left;
193 skip += copy;
194 from += copy;
195 bytes -= copy;
196 while (unlikely(!left && bytes)) {
197 iov++;
198 buf = iov->iov_base;
199 copy = min(bytes, iov->iov_len);
200 left = __copy_to_user(buf, from, copy);
201 copy -= left;
202 skip = copy;
203 from += copy;
204 bytes -= copy;
205 }
206 kunmap(page);
207 done:
208 if (skip == iov->iov_len) {
209 iov++;
210 skip = 0;
211 }
212 i->count -= wanted - bytes;
213 i->nr_segs -= iov - i->iov;
214 i->iov = iov;
215 i->iov_offset = skip;
216 return wanted - bytes;
217 }
218
copy_page_from_iter_iovec(struct page * page,size_t offset,size_t bytes,struct iov_iter * i)219 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
220 struct iov_iter *i)
221 {
222 size_t skip, copy, left, wanted;
223 const struct iovec *iov;
224 char __user *buf;
225 void *kaddr, *to;
226
227 if (unlikely(bytes > i->count))
228 bytes = i->count;
229
230 if (unlikely(!bytes))
231 return 0;
232
233 wanted = bytes;
234 iov = i->iov;
235 skip = i->iov_offset;
236 buf = iov->iov_base + skip;
237 copy = min(bytes, iov->iov_len - skip);
238
239 if (!fault_in_pages_readable(buf, copy)) {
240 kaddr = kmap_atomic(page);
241 to = kaddr + offset;
242
243 /* first chunk, usually the only one */
244 left = __copy_from_user_inatomic(to, buf, copy);
245 copy -= left;
246 skip += copy;
247 to += copy;
248 bytes -= copy;
249
250 while (unlikely(!left && bytes)) {
251 iov++;
252 buf = iov->iov_base;
253 copy = min(bytes, iov->iov_len);
254 left = __copy_from_user_inatomic(to, buf, copy);
255 copy -= left;
256 skip = copy;
257 to += copy;
258 bytes -= copy;
259 }
260 if (likely(!bytes)) {
261 kunmap_atomic(kaddr);
262 goto done;
263 }
264 offset = to - kaddr;
265 buf += copy;
266 kunmap_atomic(kaddr);
267 copy = min(bytes, iov->iov_len - skip);
268 }
269 /* Too bad - revert to non-atomic kmap */
270 kaddr = kmap(page);
271 to = kaddr + offset;
272 left = __copy_from_user(to, buf, copy);
273 copy -= left;
274 skip += copy;
275 to += copy;
276 bytes -= copy;
277 while (unlikely(!left && bytes)) {
278 iov++;
279 buf = iov->iov_base;
280 copy = min(bytes, iov->iov_len);
281 left = __copy_from_user(to, buf, copy);
282 copy -= left;
283 skip = copy;
284 to += copy;
285 bytes -= copy;
286 }
287 kunmap(page);
288 done:
289 if (skip == iov->iov_len) {
290 iov++;
291 skip = 0;
292 }
293 i->count -= wanted - bytes;
294 i->nr_segs -= iov - i->iov;
295 i->iov = iov;
296 i->iov_offset = skip;
297 return wanted - bytes;
298 }
299
300 /*
301 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
302 * bytes. For each iovec, fault in each page that constitutes the iovec.
303 *
304 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
305 * because it is an invalid address).
306 */
iov_iter_fault_in_readable(struct iov_iter * i,size_t bytes)307 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
308 {
309 size_t skip = i->iov_offset;
310 const struct iovec *iov;
311 int err;
312 struct iovec v;
313
314 if (iter_is_iovec(i)) {
315 iterate_iovec(i, bytes, v, iov, skip, ({
316 err = fault_in_multipages_readable(v.iov_base,
317 v.iov_len);
318 if (unlikely(err))
319 return err;
320 0;}))
321 }
322 return 0;
323 }
324 EXPORT_SYMBOL(iov_iter_fault_in_readable);
325
iov_iter_init(struct iov_iter * i,int direction,const struct iovec * iov,unsigned long nr_segs,size_t count)326 void iov_iter_init(struct iov_iter *i, int direction,
327 const struct iovec *iov, unsigned long nr_segs,
328 size_t count)
329 {
330 /* It will get better. Eventually... */
331 if (segment_eq(get_fs(), KERNEL_DS)) {
332 direction |= ITER_KVEC;
333 i->type = direction;
334 i->kvec = (struct kvec *)iov;
335 } else {
336 i->type = direction;
337 i->iov = iov;
338 }
339 i->nr_segs = nr_segs;
340 i->iov_offset = 0;
341 i->count = count;
342 }
343 EXPORT_SYMBOL(iov_iter_init);
344
memcpy_from_page(char * to,struct page * page,size_t offset,size_t len)345 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
346 {
347 char *from = kmap_atomic(page);
348 memcpy(to, from + offset, len);
349 kunmap_atomic(from);
350 }
351
memcpy_to_page(struct page * page,size_t offset,char * from,size_t len)352 static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
353 {
354 char *to = kmap_atomic(page);
355 memcpy(to + offset, from, len);
356 kunmap_atomic(to);
357 }
358
memzero_page(struct page * page,size_t offset,size_t len)359 static void memzero_page(struct page *page, size_t offset, size_t len)
360 {
361 char *addr = kmap_atomic(page);
362 memset(addr + offset, 0, len);
363 kunmap_atomic(addr);
364 }
365
copy_to_iter(void * addr,size_t bytes,struct iov_iter * i)366 size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i)
367 {
368 char *from = addr;
369 if (unlikely(bytes > i->count))
370 bytes = i->count;
371
372 if (unlikely(!bytes))
373 return 0;
374
375 iterate_and_advance(i, bytes, v,
376 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
377 v.iov_len),
378 memcpy_to_page(v.bv_page, v.bv_offset,
379 (from += v.bv_len) - v.bv_len, v.bv_len),
380 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
381 )
382
383 return bytes;
384 }
385 EXPORT_SYMBOL(copy_to_iter);
386
copy_from_iter(void * addr,size_t bytes,struct iov_iter * i)387 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
388 {
389 char *to = addr;
390 if (unlikely(bytes > i->count))
391 bytes = i->count;
392
393 if (unlikely(!bytes))
394 return 0;
395
396 iterate_and_advance(i, bytes, v,
397 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
398 v.iov_len),
399 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
400 v.bv_offset, v.bv_len),
401 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
402 )
403
404 return bytes;
405 }
406 EXPORT_SYMBOL(copy_from_iter);
407
copy_from_iter_nocache(void * addr,size_t bytes,struct iov_iter * i)408 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
409 {
410 char *to = addr;
411 if (unlikely(bytes > i->count))
412 bytes = i->count;
413
414 if (unlikely(!bytes))
415 return 0;
416
417 iterate_and_advance(i, bytes, v,
418 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
419 v.iov_base, v.iov_len),
420 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
421 v.bv_offset, v.bv_len),
422 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
423 )
424
425 return bytes;
426 }
427 EXPORT_SYMBOL(copy_from_iter_nocache);
428
copy_page_to_iter(struct page * page,size_t offset,size_t bytes,struct iov_iter * i)429 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
430 struct iov_iter *i)
431 {
432 if (i->type & (ITER_BVEC|ITER_KVEC)) {
433 void *kaddr = kmap_atomic(page);
434 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
435 kunmap_atomic(kaddr);
436 return wanted;
437 } else
438 return copy_page_to_iter_iovec(page, offset, bytes, i);
439 }
440 EXPORT_SYMBOL(copy_page_to_iter);
441
copy_page_from_iter(struct page * page,size_t offset,size_t bytes,struct iov_iter * i)442 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
443 struct iov_iter *i)
444 {
445 if (i->type & (ITER_BVEC|ITER_KVEC)) {
446 void *kaddr = kmap_atomic(page);
447 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
448 kunmap_atomic(kaddr);
449 return wanted;
450 } else
451 return copy_page_from_iter_iovec(page, offset, bytes, i);
452 }
453 EXPORT_SYMBOL(copy_page_from_iter);
454
iov_iter_zero(size_t bytes,struct iov_iter * i)455 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
456 {
457 if (unlikely(bytes > i->count))
458 bytes = i->count;
459
460 if (unlikely(!bytes))
461 return 0;
462
463 iterate_and_advance(i, bytes, v,
464 __clear_user(v.iov_base, v.iov_len),
465 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
466 memset(v.iov_base, 0, v.iov_len)
467 )
468
469 return bytes;
470 }
471 EXPORT_SYMBOL(iov_iter_zero);
472
iov_iter_copy_from_user_atomic(struct page * page,struct iov_iter * i,unsigned long offset,size_t bytes)473 size_t iov_iter_copy_from_user_atomic(struct page *page,
474 struct iov_iter *i, unsigned long offset, size_t bytes)
475 {
476 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
477 iterate_all_kinds(i, bytes, v,
478 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
479 v.iov_base, v.iov_len),
480 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
481 v.bv_offset, v.bv_len),
482 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
483 )
484 kunmap_atomic(kaddr);
485 return bytes;
486 }
487 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
488
iov_iter_advance(struct iov_iter * i,size_t size)489 void iov_iter_advance(struct iov_iter *i, size_t size)
490 {
491 iterate_and_advance(i, size, v, 0, 0, 0)
492 }
493 EXPORT_SYMBOL(iov_iter_advance);
494
495 /*
496 * Return the count of just the current iov_iter segment.
497 */
iov_iter_single_seg_count(const struct iov_iter * i)498 size_t iov_iter_single_seg_count(const struct iov_iter *i)
499 {
500 if (i->nr_segs == 1)
501 return i->count;
502 else if (i->type & ITER_BVEC)
503 return min(i->count, i->bvec->bv_len - i->iov_offset);
504 else
505 return min(i->count, i->iov->iov_len - i->iov_offset);
506 }
507 EXPORT_SYMBOL(iov_iter_single_seg_count);
508
iov_iter_kvec(struct iov_iter * i,int direction,const struct kvec * kvec,unsigned long nr_segs,size_t count)509 void iov_iter_kvec(struct iov_iter *i, int direction,
510 const struct kvec *kvec, unsigned long nr_segs,
511 size_t count)
512 {
513 BUG_ON(!(direction & ITER_KVEC));
514 i->type = direction;
515 i->kvec = kvec;
516 i->nr_segs = nr_segs;
517 i->iov_offset = 0;
518 i->count = count;
519 }
520 EXPORT_SYMBOL(iov_iter_kvec);
521
iov_iter_bvec(struct iov_iter * i,int direction,const struct bio_vec * bvec,unsigned long nr_segs,size_t count)522 void iov_iter_bvec(struct iov_iter *i, int direction,
523 const struct bio_vec *bvec, unsigned long nr_segs,
524 size_t count)
525 {
526 BUG_ON(!(direction & ITER_BVEC));
527 i->type = direction;
528 i->bvec = bvec;
529 i->nr_segs = nr_segs;
530 i->iov_offset = 0;
531 i->count = count;
532 }
533 EXPORT_SYMBOL(iov_iter_bvec);
534
iov_iter_alignment(const struct iov_iter * i)535 unsigned long iov_iter_alignment(const struct iov_iter *i)
536 {
537 unsigned long res = 0;
538 size_t size = i->count;
539
540 if (!size)
541 return 0;
542
543 iterate_all_kinds(i, size, v,
544 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
545 res |= v.bv_offset | v.bv_len,
546 res |= (unsigned long)v.iov_base | v.iov_len
547 )
548 return res;
549 }
550 EXPORT_SYMBOL(iov_iter_alignment);
551
iov_iter_get_pages(struct iov_iter * i,struct page ** pages,size_t maxsize,unsigned maxpages,size_t * start)552 ssize_t iov_iter_get_pages(struct iov_iter *i,
553 struct page **pages, size_t maxsize, unsigned maxpages,
554 size_t *start)
555 {
556 if (maxsize > i->count)
557 maxsize = i->count;
558
559 if (!maxsize)
560 return 0;
561
562 iterate_all_kinds(i, maxsize, v, ({
563 unsigned long addr = (unsigned long)v.iov_base;
564 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
565 int n;
566 int res;
567
568 if (len > maxpages * PAGE_SIZE)
569 len = maxpages * PAGE_SIZE;
570 addr &= ~(PAGE_SIZE - 1);
571 n = DIV_ROUND_UP(len, PAGE_SIZE);
572 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
573 if (unlikely(res < 0))
574 return res;
575 return (res == n ? len : res * PAGE_SIZE) - *start;
576 0;}),({
577 /* can't be more than PAGE_SIZE */
578 *start = v.bv_offset;
579 get_page(*pages = v.bv_page);
580 return v.bv_len;
581 }),({
582 return -EFAULT;
583 })
584 )
585 return 0;
586 }
587 EXPORT_SYMBOL(iov_iter_get_pages);
588
get_pages_array(size_t n)589 static struct page **get_pages_array(size_t n)
590 {
591 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
592 if (!p)
593 p = vmalloc(n * sizeof(struct page *));
594 return p;
595 }
596
iov_iter_get_pages_alloc(struct iov_iter * i,struct page *** pages,size_t maxsize,size_t * start)597 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
598 struct page ***pages, size_t maxsize,
599 size_t *start)
600 {
601 struct page **p;
602
603 if (maxsize > i->count)
604 maxsize = i->count;
605
606 if (!maxsize)
607 return 0;
608
609 iterate_all_kinds(i, maxsize, v, ({
610 unsigned long addr = (unsigned long)v.iov_base;
611 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
612 int n;
613 int res;
614
615 addr &= ~(PAGE_SIZE - 1);
616 n = DIV_ROUND_UP(len, PAGE_SIZE);
617 p = get_pages_array(n);
618 if (!p)
619 return -ENOMEM;
620 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
621 if (unlikely(res < 0)) {
622 kvfree(p);
623 return res;
624 }
625 *pages = p;
626 return (res == n ? len : res * PAGE_SIZE) - *start;
627 0;}),({
628 /* can't be more than PAGE_SIZE */
629 *start = v.bv_offset;
630 *pages = p = get_pages_array(1);
631 if (!p)
632 return -ENOMEM;
633 get_page(*p = v.bv_page);
634 return v.bv_len;
635 }),({
636 return -EFAULT;
637 })
638 )
639 return 0;
640 }
641 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
642
csum_and_copy_from_iter(void * addr,size_t bytes,__wsum * csum,struct iov_iter * i)643 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
644 struct iov_iter *i)
645 {
646 char *to = addr;
647 __wsum sum, next;
648 size_t off = 0;
649 if (unlikely(bytes > i->count))
650 bytes = i->count;
651
652 if (unlikely(!bytes))
653 return 0;
654
655 sum = *csum;
656 iterate_and_advance(i, bytes, v, ({
657 int err = 0;
658 next = csum_and_copy_from_user(v.iov_base,
659 (to += v.iov_len) - v.iov_len,
660 v.iov_len, 0, &err);
661 if (!err) {
662 sum = csum_block_add(sum, next, off);
663 off += v.iov_len;
664 }
665 err ? v.iov_len : 0;
666 }), ({
667 char *p = kmap_atomic(v.bv_page);
668 next = csum_partial_copy_nocheck(p + v.bv_offset,
669 (to += v.bv_len) - v.bv_len,
670 v.bv_len, 0);
671 kunmap_atomic(p);
672 sum = csum_block_add(sum, next, off);
673 off += v.bv_len;
674 }),({
675 next = csum_partial_copy_nocheck(v.iov_base,
676 (to += v.iov_len) - v.iov_len,
677 v.iov_len, 0);
678 sum = csum_block_add(sum, next, off);
679 off += v.iov_len;
680 })
681 )
682 *csum = sum;
683 return bytes;
684 }
685 EXPORT_SYMBOL(csum_and_copy_from_iter);
686
csum_and_copy_to_iter(void * addr,size_t bytes,__wsum * csum,struct iov_iter * i)687 size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum,
688 struct iov_iter *i)
689 {
690 char *from = addr;
691 __wsum sum, next;
692 size_t off = 0;
693 if (unlikely(bytes > i->count))
694 bytes = i->count;
695
696 if (unlikely(!bytes))
697 return 0;
698
699 sum = *csum;
700 iterate_and_advance(i, bytes, v, ({
701 int err = 0;
702 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
703 v.iov_base,
704 v.iov_len, 0, &err);
705 if (!err) {
706 sum = csum_block_add(sum, next, off);
707 off += v.iov_len;
708 }
709 err ? v.iov_len : 0;
710 }), ({
711 char *p = kmap_atomic(v.bv_page);
712 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
713 p + v.bv_offset,
714 v.bv_len, 0);
715 kunmap_atomic(p);
716 sum = csum_block_add(sum, next, off);
717 off += v.bv_len;
718 }),({
719 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
720 v.iov_base,
721 v.iov_len, 0);
722 sum = csum_block_add(sum, next, off);
723 off += v.iov_len;
724 })
725 )
726 *csum = sum;
727 return bytes;
728 }
729 EXPORT_SYMBOL(csum_and_copy_to_iter);
730
iov_iter_npages(const struct iov_iter * i,int maxpages)731 int iov_iter_npages(const struct iov_iter *i, int maxpages)
732 {
733 size_t size = i->count;
734 int npages = 0;
735
736 if (!size)
737 return 0;
738
739 iterate_all_kinds(i, size, v, ({
740 unsigned long p = (unsigned long)v.iov_base;
741 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
742 - p / PAGE_SIZE;
743 if (npages >= maxpages)
744 return maxpages;
745 0;}),({
746 npages++;
747 if (npages >= maxpages)
748 return maxpages;
749 }),({
750 unsigned long p = (unsigned long)v.iov_base;
751 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
752 - p / PAGE_SIZE;
753 if (npages >= maxpages)
754 return maxpages;
755 })
756 )
757 return npages;
758 }
759 EXPORT_SYMBOL(iov_iter_npages);
760
dup_iter(struct iov_iter * new,struct iov_iter * old,gfp_t flags)761 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
762 {
763 *new = *old;
764 if (new->type & ITER_BVEC)
765 return new->bvec = kmemdup(new->bvec,
766 new->nr_segs * sizeof(struct bio_vec),
767 flags);
768 else
769 /* iovec and kvec have identical layout */
770 return new->iov = kmemdup(new->iov,
771 new->nr_segs * sizeof(struct iovec),
772 flags);
773 }
774 EXPORT_SYMBOL(dup_iter);
775
import_iovec(int type,const struct iovec __user * uvector,unsigned nr_segs,unsigned fast_segs,struct iovec ** iov,struct iov_iter * i)776 int import_iovec(int type, const struct iovec __user * uvector,
777 unsigned nr_segs, unsigned fast_segs,
778 struct iovec **iov, struct iov_iter *i)
779 {
780 ssize_t n;
781 struct iovec *p;
782 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
783 *iov, &p);
784 if (n < 0) {
785 if (p != *iov)
786 kfree(p);
787 *iov = NULL;
788 return n;
789 }
790 iov_iter_init(i, type, p, nr_segs, n);
791 *iov = p == *iov ? NULL : p;
792 return 0;
793 }
794 EXPORT_SYMBOL(import_iovec);
795
796 #ifdef CONFIG_COMPAT
797 #include <linux/compat.h>
798
compat_import_iovec(int type,const struct compat_iovec __user * uvector,unsigned nr_segs,unsigned fast_segs,struct iovec ** iov,struct iov_iter * i)799 int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
800 unsigned nr_segs, unsigned fast_segs,
801 struct iovec **iov, struct iov_iter *i)
802 {
803 ssize_t n;
804 struct iovec *p;
805 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
806 *iov, &p);
807 if (n < 0) {
808 if (p != *iov)
809 kfree(p);
810 *iov = NULL;
811 return n;
812 }
813 iov_iter_init(i, type, p, nr_segs, n);
814 *iov = p == *iov ? NULL : p;
815 return 0;
816 }
817 #endif
818
import_single_range(int rw,void __user * buf,size_t len,struct iovec * iov,struct iov_iter * i)819 int import_single_range(int rw, void __user *buf, size_t len,
820 struct iovec *iov, struct iov_iter *i)
821 {
822 if (len > MAX_RW_COUNT)
823 len = MAX_RW_COUNT;
824 if (unlikely(!access_ok(!rw, buf, len)))
825 return -EFAULT;
826
827 iov->iov_base = buf;
828 iov->iov_len = len;
829 iov_iter_init(i, rw, iov, 1, len);
830 return 0;
831 }
832