Lines Matching refs:i
20 #define iterate_iovec(i, n, base, len, off, __p, STEP) { \ argument
22 size_t skip = i->iov_offset; \
37 i->iov_offset = skip; \
41 #define iterate_bvec(i, n, base, len, off, p, STEP) { \ argument
43 unsigned skip = i->iov_offset; \
65 i->iov_offset = skip; \
69 #define iterate_xarray(i, n, base, len, __off, STEP) { \ argument
73 loff_t start = i->xarray_start + i->iov_offset; \
78 XA_STATE(xas, i->xarray, index); \
107 i->iov_offset += __off; \
111 #define __iterate_and_advance(i, n, base, len, off, I, K) { \ argument
112 if (unlikely(i->count < n)) \
113 n = i->count; \
115 if (likely(iter_is_iovec(i))) { \
116 const struct iovec *iov = i->iov; \
119 iterate_iovec(i, n, base, len, off, \
121 i->nr_segs -= iov - i->iov; \
122 i->iov = iov; \
123 } else if (iov_iter_is_bvec(i)) { \
124 const struct bio_vec *bvec = i->bvec; \
127 iterate_bvec(i, n, base, len, off, \
129 i->nr_segs -= bvec - i->bvec; \
130 i->bvec = bvec; \
131 } else if (iov_iter_is_kvec(i)) { \
132 const struct kvec *kvec = i->kvec; \
135 iterate_iovec(i, n, base, len, off, \
137 i->nr_segs -= kvec - i->kvec; \
138 i->kvec = kvec; \
139 } else if (iov_iter_is_xarray(i)) { \
142 iterate_xarray(i, n, base, len, off, \
145 i->count -= n; \
148 #define iterate_and_advance(i, n, base, len, off, I, K) \ argument
149 __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0))
174 struct iov_iter *i) in copy_page_to_iter_iovec() argument
181 if (unlikely(bytes > i->count)) in copy_page_to_iter_iovec()
182 bytes = i->count; in copy_page_to_iter_iovec()
189 iov = i->iov; in copy_page_to_iter_iovec()
190 skip = i->iov_offset; in copy_page_to_iter_iovec()
250 i->count -= wanted - bytes; in copy_page_to_iter_iovec()
251 i->nr_segs -= iov - i->iov; in copy_page_to_iter_iovec()
252 i->iov = iov; in copy_page_to_iter_iovec()
253 i->iov_offset = skip; in copy_page_to_iter_iovec()
258 struct iov_iter *i) in copy_page_from_iter_iovec() argument
265 if (unlikely(bytes > i->count)) in copy_page_from_iter_iovec()
266 bytes = i->count; in copy_page_from_iter_iovec()
273 iov = i->iov; in copy_page_from_iter_iovec()
274 skip = i->iov_offset; in copy_page_from_iter_iovec()
334 i->count -= wanted - bytes; in copy_page_from_iter_iovec()
335 i->nr_segs -= iov - i->iov; in copy_page_from_iter_iovec()
336 i->iov = iov; in copy_page_from_iter_iovec()
337 i->iov_offset = skip; in copy_page_from_iter_iovec()
342 static bool sanity(const struct iov_iter *i) in sanity() argument
344 struct pipe_inode_info *pipe = i->pipe; in sanity()
349 unsigned int i_head = i->head; in sanity()
352 if (i->iov_offset) { in sanity()
360 if (unlikely(p->offset + p->len != i->iov_offset)) in sanity()
368 printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset); in sanity()
381 #define sanity(i) true argument
385 struct iov_iter *i) in copy_page_to_iter_pipe() argument
387 struct pipe_inode_info *pipe = i->pipe; in copy_page_to_iter_pipe()
391 unsigned int i_head = i->head; in copy_page_to_iter_pipe()
394 if (unlikely(bytes > i->count)) in copy_page_to_iter_pipe()
395 bytes = i->count; in copy_page_to_iter_pipe()
400 if (!sanity(i)) in copy_page_to_iter_pipe()
403 off = i->iov_offset; in copy_page_to_iter_pipe()
409 i->iov_offset += bytes; in copy_page_to_iter_pipe()
426 i->iov_offset = offset + bytes; in copy_page_to_iter_pipe()
427 i->head = i_head; in copy_page_to_iter_pipe()
429 i->count -= bytes; in copy_page_to_iter_pipe()
446 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) in fault_in_iov_iter_readable() argument
448 if (iter_is_iovec(i)) { in fault_in_iov_iter_readable()
449 size_t count = min(size, iov_iter_count(i)); in fault_in_iov_iter_readable()
454 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { in fault_in_iov_iter_readable()
485 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) in fault_in_iov_iter_writeable() argument
487 if (iter_is_iovec(i)) { in fault_in_iov_iter_writeable()
488 size_t count = min(size, iov_iter_count(i)); in fault_in_iov_iter_writeable()
493 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { in fault_in_iov_iter_writeable()
510 void iov_iter_init(struct iov_iter *i, unsigned int direction, in iov_iter_init() argument
515 *i = (struct iov_iter) { in iov_iter_init()
532 static inline void data_start(const struct iov_iter *i, in data_start() argument
535 unsigned int p_mask = i->pipe->ring_size - 1; in data_start()
536 unsigned int iter_head = i->head; in data_start()
537 size_t off = i->iov_offset; in data_start()
539 if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || in data_start()
548 static size_t push_pipe(struct iov_iter *i, size_t size, in push_pipe() argument
551 struct pipe_inode_info *pipe = i->pipe; in push_pipe()
558 if (unlikely(size > i->count)) in push_pipe()
559 size = i->count; in push_pipe()
564 data_start(i, &iter_head, &off); in push_pipe()
598 struct iov_iter *i) in copy_pipe_to_iter() argument
600 struct pipe_inode_info *pipe = i->pipe; in copy_pipe_to_iter()
605 if (!sanity(i)) in copy_pipe_to_iter()
608 bytes = n = push_pipe(i, bytes, &i_head, &off); in copy_pipe_to_iter()
614 i->head = i_head; in copy_pipe_to_iter()
615 i->iov_offset = off + chunk; in copy_pipe_to_iter()
621 i->count -= bytes; in copy_pipe_to_iter()
633 struct iov_iter *i, __wsum *sump) in csum_and_copy_to_pipe_iter() argument
635 struct pipe_inode_info *pipe = i->pipe; in csum_and_copy_to_pipe_iter()
642 if (!sanity(i)) in csum_and_copy_to_pipe_iter()
645 bytes = push_pipe(i, bytes, &i_head, &r); in csum_and_copy_to_pipe_iter()
651 i->head = i_head; in csum_and_copy_to_pipe_iter()
652 i->iov_offset = r + chunk; in csum_and_copy_to_pipe_iter()
659 i->count -= off; in csum_and_copy_to_pipe_iter()
663 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_to_iter() argument
665 if (unlikely(iov_iter_is_pipe(i))) in _copy_to_iter()
666 return copy_pipe_to_iter(addr, bytes, i); in _copy_to_iter()
667 if (iter_is_iovec(i)) in _copy_to_iter()
669 iterate_and_advance(i, bytes, base, len, off, in _copy_to_iter()
689 struct iov_iter *i) in copy_mc_pipe_to_iter() argument
691 struct pipe_inode_info *pipe = i->pipe; in copy_mc_pipe_to_iter()
697 if (!sanity(i)) in copy_mc_pipe_to_iter()
700 n = push_pipe(i, bytes, &i_head, &off); in copy_mc_pipe_to_iter()
709 i->head = i_head; in copy_mc_pipe_to_iter()
710 i->iov_offset = off + chunk; in copy_mc_pipe_to_iter()
723 i->count -= xfer; in copy_mc_pipe_to_iter()
752 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_mc_to_iter() argument
754 if (unlikely(iov_iter_is_pipe(i))) in _copy_mc_to_iter()
755 return copy_mc_pipe_to_iter(addr, bytes, i); in _copy_mc_to_iter()
756 if (iter_is_iovec(i)) in _copy_mc_to_iter()
758 __iterate_and_advance(i, bytes, base, len, off, in _copy_mc_to_iter()
768 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter() argument
770 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter()
774 if (iter_is_iovec(i)) in _copy_from_iter()
776 iterate_and_advance(i, bytes, base, len, off, in _copy_from_iter()
785 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_nocache() argument
787 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter_nocache()
791 iterate_and_advance(i, bytes, base, len, off, in _copy_from_iter_nocache()
817 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_flushcache() argument
819 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter_flushcache()
823 iterate_and_advance(i, bytes, base, len, off, in _copy_from_iter_flushcache()
858 struct iov_iter *i) in __copy_page_to_iter() argument
860 if (likely(iter_is_iovec(i))) in __copy_page_to_iter()
861 return copy_page_to_iter_iovec(page, offset, bytes, i); in __copy_page_to_iter()
862 if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { in __copy_page_to_iter()
864 size_t wanted = _copy_to_iter(kaddr + offset, bytes, i); in __copy_page_to_iter()
868 if (iov_iter_is_pipe(i)) in __copy_page_to_iter()
869 return copy_page_to_iter_pipe(page, offset, bytes, i); in __copy_page_to_iter()
870 if (unlikely(iov_iter_is_discard(i))) { in __copy_page_to_iter()
871 if (unlikely(i->count < bytes)) in __copy_page_to_iter()
872 bytes = i->count; in __copy_page_to_iter()
873 i->count -= bytes; in __copy_page_to_iter()
881 struct iov_iter *i) in copy_page_to_iter() argument
890 min(bytes, (size_t)PAGE_SIZE - offset), i); in copy_page_to_iter()
906 struct iov_iter *i) in copy_page_from_iter() argument
910 if (likely(iter_is_iovec(i))) in copy_page_from_iter()
911 return copy_page_from_iter_iovec(page, offset, bytes, i); in copy_page_from_iter()
912 if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { in copy_page_from_iter()
914 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); in copy_page_from_iter()
923 static size_t pipe_zero(size_t bytes, struct iov_iter *i) in pipe_zero() argument
925 struct pipe_inode_info *pipe = i->pipe; in pipe_zero()
930 if (!sanity(i)) in pipe_zero()
933 bytes = n = push_pipe(i, bytes, &i_head, &off); in pipe_zero()
942 i->head = i_head; in pipe_zero()
943 i->iov_offset = off + chunk; in pipe_zero()
948 i->count -= bytes; in pipe_zero()
952 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) in iov_iter_zero() argument
954 if (unlikely(iov_iter_is_pipe(i))) in iov_iter_zero()
955 return pipe_zero(bytes, i); in iov_iter_zero()
956 iterate_and_advance(i, bytes, base, len, count, in iov_iter_zero()
966 struct iov_iter *i) in copy_page_from_iter_atomic() argument
973 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in copy_page_from_iter_atomic()
978 iterate_and_advance(i, bytes, base, len, off, in copy_page_from_iter_atomic()
987 static inline void pipe_truncate(struct iov_iter *i) in pipe_truncate() argument
989 struct pipe_inode_info *pipe = i->pipe; in pipe_truncate()
996 unsigned int i_head = i->head; in pipe_truncate()
997 size_t off = i->iov_offset; in pipe_truncate()
1013 static void pipe_advance(struct iov_iter *i, size_t size) in pipe_advance() argument
1015 struct pipe_inode_info *pipe = i->pipe; in pipe_advance()
1019 unsigned int i_head = i->head; in pipe_advance()
1020 size_t off = i->iov_offset, left = size; in pipe_advance()
1031 i->head = i_head; in pipe_advance()
1032 i->iov_offset = buf->offset + left; in pipe_advance()
1034 i->count -= size; in pipe_advance()
1036 pipe_truncate(i); in pipe_advance()
1039 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) in iov_iter_bvec_advance() argument
1043 bi.bi_size = i->count; in iov_iter_bvec_advance()
1044 bi.bi_bvec_done = i->iov_offset; in iov_iter_bvec_advance()
1046 bvec_iter_advance(i->bvec, &bi, size); in iov_iter_bvec_advance()
1048 i->bvec += bi.bi_idx; in iov_iter_bvec_advance()
1049 i->nr_segs -= bi.bi_idx; in iov_iter_bvec_advance()
1050 i->count = bi.bi_size; in iov_iter_bvec_advance()
1051 i->iov_offset = bi.bi_bvec_done; in iov_iter_bvec_advance()
1054 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) in iov_iter_iovec_advance() argument
1058 if (!i->count) in iov_iter_iovec_advance()
1060 i->count -= size; in iov_iter_iovec_advance()
1062 size += i->iov_offset; // from beginning of current segment in iov_iter_iovec_advance()
1063 for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) { in iov_iter_iovec_advance()
1068 i->iov_offset = size; in iov_iter_iovec_advance()
1069 i->nr_segs -= iov - i->iov; in iov_iter_iovec_advance()
1070 i->iov = iov; in iov_iter_iovec_advance()
1073 void iov_iter_advance(struct iov_iter *i, size_t size) in iov_iter_advance() argument
1075 if (unlikely(i->count < size)) in iov_iter_advance()
1076 size = i->count; in iov_iter_advance()
1077 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { in iov_iter_advance()
1079 iov_iter_iovec_advance(i, size); in iov_iter_advance()
1080 } else if (iov_iter_is_bvec(i)) { in iov_iter_advance()
1081 iov_iter_bvec_advance(i, size); in iov_iter_advance()
1082 } else if (iov_iter_is_pipe(i)) { in iov_iter_advance()
1083 pipe_advance(i, size); in iov_iter_advance()
1084 } else if (unlikely(iov_iter_is_xarray(i))) { in iov_iter_advance()
1085 i->iov_offset += size; in iov_iter_advance()
1086 i->count -= size; in iov_iter_advance()
1087 } else if (iov_iter_is_discard(i)) { in iov_iter_advance()
1088 i->count -= size; in iov_iter_advance()
1093 void iov_iter_revert(struct iov_iter *i, size_t unroll) in iov_iter_revert() argument
1099 i->count += unroll; in iov_iter_revert()
1100 if (unlikely(iov_iter_is_pipe(i))) { in iov_iter_revert()
1101 struct pipe_inode_info *pipe = i->pipe; in iov_iter_revert()
1103 unsigned int i_head = i->head; in iov_iter_revert()
1104 size_t off = i->iov_offset; in iov_iter_revert()
1113 if (!unroll && i_head == i->start_head) { in iov_iter_revert()
1121 i->iov_offset = off; in iov_iter_revert()
1122 i->head = i_head; in iov_iter_revert()
1123 pipe_truncate(i); in iov_iter_revert()
1126 if (unlikely(iov_iter_is_discard(i))) in iov_iter_revert()
1128 if (unroll <= i->iov_offset) { in iov_iter_revert()
1129 i->iov_offset -= unroll; in iov_iter_revert()
1132 unroll -= i->iov_offset; in iov_iter_revert()
1133 if (iov_iter_is_xarray(i)) { in iov_iter_revert()
1138 } else if (iov_iter_is_bvec(i)) { in iov_iter_revert()
1139 const struct bio_vec *bvec = i->bvec; in iov_iter_revert()
1142 i->nr_segs++; in iov_iter_revert()
1144 i->bvec = bvec; in iov_iter_revert()
1145 i->iov_offset = n - unroll; in iov_iter_revert()
1151 const struct iovec *iov = i->iov; in iov_iter_revert()
1154 i->nr_segs++; in iov_iter_revert()
1156 i->iov = iov; in iov_iter_revert()
1157 i->iov_offset = n - unroll; in iov_iter_revert()
1169 size_t iov_iter_single_seg_count(const struct iov_iter *i) in iov_iter_single_seg_count() argument
1171 if (i->nr_segs > 1) { in iov_iter_single_seg_count()
1172 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_single_seg_count()
1173 return min(i->count, i->iov->iov_len - i->iov_offset); in iov_iter_single_seg_count()
1174 if (iov_iter_is_bvec(i)) in iov_iter_single_seg_count()
1175 return min(i->count, i->bvec->bv_len - i->iov_offset); in iov_iter_single_seg_count()
1177 return i->count; in iov_iter_single_seg_count()
1181 void iov_iter_kvec(struct iov_iter *i, unsigned int direction, in iov_iter_kvec() argument
1186 *i = (struct iov_iter){ in iov_iter_kvec()
1197 void iov_iter_bvec(struct iov_iter *i, unsigned int direction, in iov_iter_bvec() argument
1202 *i = (struct iov_iter){ in iov_iter_bvec()
1213 void iov_iter_pipe(struct iov_iter *i, unsigned int direction, in iov_iter_pipe() argument
1219 *i = (struct iov_iter){ in iov_iter_pipe()
1244 void iov_iter_xarray(struct iov_iter *i, unsigned int direction, in iov_iter_xarray() argument
1248 *i = (struct iov_iter) { in iov_iter_xarray()
1268 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) in iov_iter_discard() argument
1271 *i = (struct iov_iter){ in iov_iter_discard()
1280 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) in iov_iter_alignment_iovec() argument
1283 size_t size = i->count; in iov_iter_alignment_iovec()
1284 size_t skip = i->iov_offset; in iov_iter_alignment_iovec()
1287 for (k = 0; k < i->nr_segs; k++, skip = 0) { in iov_iter_alignment_iovec()
1288 size_t len = i->iov[k].iov_len - skip; in iov_iter_alignment_iovec()
1290 res |= (unsigned long)i->iov[k].iov_base + skip; in iov_iter_alignment_iovec()
1302 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) in iov_iter_alignment_bvec() argument
1305 size_t size = i->count; in iov_iter_alignment_bvec()
1306 unsigned skip = i->iov_offset; in iov_iter_alignment_bvec()
1309 for (k = 0; k < i->nr_segs; k++, skip = 0) { in iov_iter_alignment_bvec()
1310 size_t len = i->bvec[k].bv_len - skip; in iov_iter_alignment_bvec()
1311 res |= (unsigned long)i->bvec[k].bv_offset + skip; in iov_iter_alignment_bvec()
1322 unsigned long iov_iter_alignment(const struct iov_iter *i) in iov_iter_alignment() argument
1325 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_alignment()
1326 return iov_iter_alignment_iovec(i); in iov_iter_alignment()
1328 if (iov_iter_is_bvec(i)) in iov_iter_alignment()
1329 return iov_iter_alignment_bvec(i); in iov_iter_alignment()
1331 if (iov_iter_is_pipe(i)) { in iov_iter_alignment()
1332 unsigned int p_mask = i->pipe->ring_size - 1; in iov_iter_alignment()
1333 size_t size = i->count; in iov_iter_alignment()
1335 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) in iov_iter_alignment()
1336 return size | i->iov_offset; in iov_iter_alignment()
1340 if (iov_iter_is_xarray(i)) in iov_iter_alignment()
1341 return (i->xarray_start + i->iov_offset) | i->count; in iov_iter_alignment()
1347 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) in iov_iter_gap_alignment() argument
1351 size_t size = i->count; in iov_iter_gap_alignment()
1354 if (WARN_ON(!iter_is_iovec(i))) in iov_iter_gap_alignment()
1357 for (k = 0; k < i->nr_segs; k++) { in iov_iter_gap_alignment()
1358 if (i->iov[k].iov_len) { in iov_iter_gap_alignment()
1359 unsigned long base = (unsigned long)i->iov[k].iov_base; in iov_iter_gap_alignment()
1362 v = base + i->iov[k].iov_len; in iov_iter_gap_alignment()
1363 if (size <= i->iov[k].iov_len) in iov_iter_gap_alignment()
1365 size -= i->iov[k].iov_len; in iov_iter_gap_alignment()
1372 static inline ssize_t __pipe_get_pages(struct iov_iter *i, in __pipe_get_pages() argument
1378 struct pipe_inode_info *pipe = i->pipe; in __pipe_get_pages()
1380 ssize_t n = push_pipe(i, maxsize, &iter_head, start); in __pipe_get_pages()
1395 static ssize_t pipe_get_pages(struct iov_iter *i, in pipe_get_pages() argument
1402 if (!sanity(i)) in pipe_get_pages()
1405 data_start(i, &iter_head, start); in pipe_get_pages()
1407 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); in pipe_get_pages()
1410 return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start); in pipe_get_pages()
1440 static ssize_t iter_xarray_get_pages(struct iov_iter *i, in iter_xarray_get_pages() argument
1452 pos = i->xarray_start + i->iov_offset; in iter_xarray_get_pages()
1469 nr = iter_xarray_populate_pages(pages, i->xarray, index, count); in iter_xarray_get_pages()
1477 static unsigned long first_iovec_segment(const struct iov_iter *i, in first_iovec_segment() argument
1484 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { in first_iovec_segment()
1485 unsigned long addr = (unsigned long)i->iov[k].iov_base + skip; in first_iovec_segment()
1486 size_t len = i->iov[k].iov_len - skip; in first_iovec_segment()
1502 static struct page *first_bvec_segment(const struct iov_iter *i, in first_bvec_segment() argument
1507 size_t skip = i->iov_offset, len; in first_bvec_segment()
1509 len = i->bvec->bv_len - skip; in first_bvec_segment()
1512 skip += i->bvec->bv_offset; in first_bvec_segment()
1513 page = i->bvec->bv_page + skip / PAGE_SIZE; in first_bvec_segment()
1521 ssize_t iov_iter_get_pages(struct iov_iter *i, in iov_iter_get_pages() argument
1528 if (maxsize > i->count) in iov_iter_get_pages()
1529 maxsize = i->count; in iov_iter_get_pages()
1533 if (likely(iter_is_iovec(i))) { in iov_iter_get_pages()
1537 if (iov_iter_rw(i) != WRITE) in iov_iter_get_pages()
1539 if (i->nofault) in iov_iter_get_pages()
1542 addr = first_iovec_segment(i, &len, start, maxsize, maxpages); in iov_iter_get_pages()
1549 if (iov_iter_is_bvec(i)) { in iov_iter_get_pages()
1552 page = first_bvec_segment(i, &len, start, maxsize, maxpages); in iov_iter_get_pages()
1558 if (iov_iter_is_pipe(i)) in iov_iter_get_pages()
1559 return pipe_get_pages(i, pages, maxsize, maxpages, start); in iov_iter_get_pages()
1560 if (iov_iter_is_xarray(i)) in iov_iter_get_pages()
1561 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); in iov_iter_get_pages()
1571 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, in pipe_get_pages_alloc() argument
1579 if (!sanity(i)) in pipe_get_pages_alloc()
1582 data_start(i, &iter_head, start); in pipe_get_pages_alloc()
1584 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); in pipe_get_pages_alloc()
1593 n = __pipe_get_pages(i, maxsize, p, iter_head, start); in pipe_get_pages_alloc()
1601 static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i, in iter_xarray_get_pages_alloc() argument
1614 pos = i->xarray_start + i->iov_offset; in iter_xarray_get_pages_alloc()
1633 nr = iter_xarray_populate_pages(p, i->xarray, index, count); in iter_xarray_get_pages_alloc()
1640 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, in iov_iter_get_pages_alloc() argument
1648 if (maxsize > i->count) in iov_iter_get_pages_alloc()
1649 maxsize = i->count; in iov_iter_get_pages_alloc()
1653 if (likely(iter_is_iovec(i))) { in iov_iter_get_pages_alloc()
1657 if (iov_iter_rw(i) != WRITE) in iov_iter_get_pages_alloc()
1659 if (i->nofault) in iov_iter_get_pages_alloc()
1662 addr = first_iovec_segment(i, &len, start, maxsize, ~0U); in iov_iter_get_pages_alloc()
1676 if (iov_iter_is_bvec(i)) { in iov_iter_get_pages_alloc()
1679 page = first_bvec_segment(i, &len, start, maxsize, ~0U); in iov_iter_get_pages_alloc()
1688 if (iov_iter_is_pipe(i)) in iov_iter_get_pages_alloc()
1689 return pipe_get_pages_alloc(i, pages, maxsize, start); in iov_iter_get_pages_alloc()
1690 if (iov_iter_is_xarray(i)) in iov_iter_get_pages_alloc()
1691 return iter_xarray_get_pages_alloc(i, pages, maxsize, start); in iov_iter_get_pages_alloc()
1697 struct iov_iter *i) in csum_and_copy_from_iter() argument
1701 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in csum_and_copy_from_iter()
1705 iterate_and_advance(i, bytes, base, len, off, ({ in csum_and_copy_from_iter()
1719 struct iov_iter *i) in csum_and_copy_to_iter() argument
1724 if (unlikely(iov_iter_is_discard(i))) { in csum_and_copy_to_iter()
1730 if (unlikely(iov_iter_is_pipe(i))) in csum_and_copy_to_iter()
1731 bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum); in csum_and_copy_to_iter()
1732 else iterate_and_advance(i, bytes, base, len, off, ({ in csum_and_copy_to_iter()
1747 struct iov_iter *i) in hash_and_copy_to_iter() argument
1754 copied = copy_to_iter(addr, bytes, i); in hash_and_copy_to_iter()
1765 static int iov_npages(const struct iov_iter *i, int maxpages) in iov_npages() argument
1767 size_t skip = i->iov_offset, size = i->count; in iov_npages()
1771 for (p = i->iov; size; skip = 0, p++) { in iov_npages()
1785 static int bvec_npages(const struct iov_iter *i, int maxpages) in bvec_npages() argument
1787 size_t skip = i->iov_offset, size = i->count; in bvec_npages()
1791 for (p = i->bvec; size; skip = 0, p++) { in bvec_npages()
1803 int iov_iter_npages(const struct iov_iter *i, int maxpages) in iov_iter_npages() argument
1805 if (unlikely(!i->count)) in iov_iter_npages()
1808 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) in iov_iter_npages()
1809 return iov_npages(i, maxpages); in iov_iter_npages()
1810 if (iov_iter_is_bvec(i)) in iov_iter_npages()
1811 return bvec_npages(i, maxpages); in iov_iter_npages()
1812 if (iov_iter_is_pipe(i)) { in iov_iter_npages()
1817 if (!sanity(i)) in iov_iter_npages()
1820 data_start(i, &iter_head, &off); in iov_iter_npages()
1822 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); in iov_iter_npages()
1825 if (iov_iter_is_xarray(i)) { in iov_iter_npages()
1826 unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; in iov_iter_npages()
1827 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); in iov_iter_npages()
1860 int ret = -EFAULT, i; in copy_compat_iovec_from_user() local
1865 for (i = 0; i < nr_segs; i++) { in copy_compat_iovec_from_user()
1869 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); in copy_compat_iovec_from_user()
1870 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); in copy_compat_iovec_from_user()
1877 iov[i].iov_base = compat_ptr(buf); in copy_compat_iovec_from_user()
1878 iov[i].iov_len = len; in copy_compat_iovec_from_user()
1939 struct iov_iter *i, bool compat) in __import_iovec() argument
1976 iov_iter_init(i, type, iov, nr_segs, total_len); in __import_iovec()
2008 struct iovec **iovp, struct iov_iter *i) in import_iovec() argument
2010 return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, in import_iovec()
2016 struct iovec *iov, struct iov_iter *i) in import_single_range() argument
2025 iov_iter_init(i, rw, iov, 1, len); in import_single_range()
2042 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) in iov_iter_restore() argument
2044 if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && in iov_iter_restore()
2045 !iov_iter_is_kvec(i)) in iov_iter_restore()
2047 i->iov_offset = state->iov_offset; in iov_iter_restore()
2048 i->count = state->count; in iov_iter_restore()
2059 if (iov_iter_is_bvec(i)) in iov_iter_restore()
2060 i->bvec -= state->nr_segs - i->nr_segs; in iov_iter_restore()
2062 i->iov -= state->nr_segs - i->nr_segs; in iov_iter_restore()
2063 i->nr_segs = state->nr_segs; in iov_iter_restore()