Lines Matching full:i
18 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ argument
21 __p = i->iov; \
46 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ argument
48 __p = i->kvec; \
69 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ argument
74 for_each_bvec(__v, i->bvec, __bi, __start) { \
81 #define iterate_all_kinds(i, n, v, I, B, K) { \ argument
83 size_t skip = i->iov_offset; \
84 if (unlikely(i->type & ITER_BVEC)) { \
87 iterate_bvec(i, n, v, __bi, skip, (B)) \
88 } else if (unlikely(i->type & ITER_KVEC)) { \
91 iterate_kvec(i, n, v, kvec, skip, (K)) \
92 } else if (unlikely(i->type & ITER_DISCARD)) { \
96 iterate_iovec(i, n, v, iov, skip, (I)) \
101 #define iterate_and_advance(i, n, v, I, B, K) { \ argument
102 if (unlikely(i->count < n)) \
103 n = i->count; \
104 if (i->count) { \
105 size_t skip = i->iov_offset; \
106 if (unlikely(i->type & ITER_BVEC)) { \
107 const struct bio_vec *bvec = i->bvec; \
110 iterate_bvec(i, n, v, __bi, skip, (B)) \
111 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
112 i->nr_segs -= i->bvec - bvec; \
114 } else if (unlikely(i->type & ITER_KVEC)) { \
117 iterate_kvec(i, n, v, kvec, skip, (K)) \
122 i->nr_segs -= kvec - i->kvec; \
123 i->kvec = kvec; \
124 } else if (unlikely(i->type & ITER_DISCARD)) { \
129 iterate_iovec(i, n, v, iov, skip, (I)) \
134 i->nr_segs -= iov - i->iov; \
135 i->iov = iov; \
137 i->count -= n; \
138 i->iov_offset = skip; \
165 struct iov_iter *i) in copy_page_to_iter_iovec() argument
172 if (unlikely(bytes > i->count)) in copy_page_to_iter_iovec()
173 bytes = i->count; in copy_page_to_iter_iovec()
180 iov = i->iov; in copy_page_to_iter_iovec()
181 skip = i->iov_offset; in copy_page_to_iter_iovec()
241 i->count -= wanted - bytes; in copy_page_to_iter_iovec()
242 i->nr_segs -= iov - i->iov; in copy_page_to_iter_iovec()
243 i->iov = iov; in copy_page_to_iter_iovec()
244 i->iov_offset = skip; in copy_page_to_iter_iovec()
249 struct iov_iter *i) in copy_page_from_iter_iovec() argument
256 if (unlikely(bytes > i->count)) in copy_page_from_iter_iovec()
257 bytes = i->count; in copy_page_from_iter_iovec()
264 iov = i->iov; in copy_page_from_iter_iovec()
265 skip = i->iov_offset; in copy_page_from_iter_iovec()
325 i->count -= wanted - bytes; in copy_page_from_iter_iovec()
326 i->nr_segs -= iov - i->iov; in copy_page_from_iter_iovec()
327 i->iov = iov; in copy_page_from_iter_iovec()
328 i->iov_offset = skip; in copy_page_from_iter_iovec()
333 static bool sanity(const struct iov_iter *i) in sanity() argument
335 struct pipe_inode_info *pipe = i->pipe; in sanity()
340 unsigned int i_head = i->head; in sanity()
343 if (i->iov_offset) { in sanity()
351 if (unlikely(p->offset + p->len != i->iov_offset)) in sanity()
359 printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset); in sanity()
372 #define sanity(i) true argument
376 struct iov_iter *i) in copy_page_to_iter_pipe() argument
378 struct pipe_inode_info *pipe = i->pipe; in copy_page_to_iter_pipe()
382 unsigned int i_head = i->head; in copy_page_to_iter_pipe()
385 if (unlikely(bytes > i->count)) in copy_page_to_iter_pipe()
386 bytes = i->count; in copy_page_to_iter_pipe()
391 if (!sanity(i)) in copy_page_to_iter_pipe()
394 off = i->iov_offset; in copy_page_to_iter_pipe()
400 i->iov_offset += bytes; in copy_page_to_iter_pipe()
417 i->iov_offset = offset + bytes; in copy_page_to_iter_pipe()
418 i->head = i_head; in copy_page_to_iter_pipe()
420 i->count -= bytes; in copy_page_to_iter_pipe()
428 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
431 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) in iov_iter_fault_in_readable() argument
433 size_t skip = i->iov_offset; in iov_iter_fault_in_readable()
438 if (iter_is_iovec(i)) { in iov_iter_fault_in_readable()
439 iterate_iovec(i, bytes, v, iov, skip, ({ in iov_iter_fault_in_readable()
449 void iov_iter_init(struct iov_iter *i, unsigned int direction, in iov_iter_init() argument
458 i->type = ITER_KVEC | direction; in iov_iter_init()
459 i->kvec = (struct kvec *)iov; in iov_iter_init()
461 i->type = ITER_IOVEC | direction; in iov_iter_init()
462 i->iov = iov; in iov_iter_init()
464 i->nr_segs = nr_segs; in iov_iter_init()
465 i->iov_offset = 0; in iov_iter_init()
466 i->count = count; in iov_iter_init()
482 static inline void data_start(const struct iov_iter *i, in data_start() argument
485 unsigned int p_mask = i->pipe->ring_size - 1; in data_start()
486 unsigned int iter_head = i->head; in data_start()
487 size_t off = i->iov_offset; in data_start()
489 if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) || in data_start()
498 static size_t push_pipe(struct iov_iter *i, size_t size, in push_pipe() argument
501 struct pipe_inode_info *pipe = i->pipe; in push_pipe()
508 if (unlikely(size > i->count)) in push_pipe()
509 size = i->count; in push_pipe()
514 data_start(i, &iter_head, &off); in push_pipe()
548 struct iov_iter *i) in copy_pipe_to_iter() argument
550 struct pipe_inode_info *pipe = i->pipe; in copy_pipe_to_iter()
555 if (!sanity(i)) in copy_pipe_to_iter()
558 bytes = n = push_pipe(i, bytes, &i_head, &off); in copy_pipe_to_iter()
564 i->head = i_head; in copy_pipe_to_iter()
565 i->iov_offset = off + chunk; in copy_pipe_to_iter()
571 i->count -= bytes; in copy_pipe_to_iter()
584 struct iov_iter *i) in csum_and_copy_to_pipe_iter() argument
586 struct pipe_inode_info *pipe = i->pipe; in csum_and_copy_to_pipe_iter()
593 if (!sanity(i)) in csum_and_copy_to_pipe_iter()
596 bytes = n = push_pipe(i, bytes, &i_head, &r); in csum_and_copy_to_pipe_iter()
604 i->head = i_head; in csum_and_copy_to_pipe_iter()
605 i->iov_offset = r + chunk; in csum_and_copy_to_pipe_iter()
612 i->count -= bytes; in csum_and_copy_to_pipe_iter()
618 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_to_iter() argument
621 if (unlikely(iov_iter_is_pipe(i))) in _copy_to_iter()
622 return copy_pipe_to_iter(addr, bytes, i); in _copy_to_iter()
623 if (iter_is_iovec(i)) in _copy_to_iter()
625 iterate_and_advance(i, bytes, v, in _copy_to_iter()
660 struct iov_iter *i) in copy_mc_pipe_to_iter() argument
662 struct pipe_inode_info *pipe = i->pipe; in copy_mc_pipe_to_iter()
667 if (!sanity(i)) in copy_mc_pipe_to_iter()
670 bytes = n = push_pipe(i, bytes, &i_head, &off); in copy_mc_pipe_to_iter()
679 i->head = i_head; in copy_mc_pipe_to_iter()
680 i->iov_offset = off + chunk - rem; in copy_mc_pipe_to_iter()
689 i->count -= xfer; in copy_mc_pipe_to_iter()
716 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_mc_to_iter() argument
721 if (unlikely(iov_iter_is_pipe(i))) in _copy_mc_to_iter()
722 return copy_mc_pipe_to_iter(addr, bytes, i); in _copy_mc_to_iter()
723 if (iter_is_iovec(i)) in _copy_mc_to_iter()
725 iterate_and_advance(i, bytes, v, in _copy_mc_to_iter()
753 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter() argument
756 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter()
760 if (iter_is_iovec(i)) in _copy_from_iter()
762 iterate_and_advance(i, bytes, v, in _copy_from_iter()
773 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_full() argument
776 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter_full()
780 if (unlikely(i->count < bytes)) in _copy_from_iter_full()
783 if (iter_is_iovec(i)) in _copy_from_iter_full()
785 iterate_all_kinds(i, bytes, v, ({ in _copy_from_iter_full()
795 iov_iter_advance(i, bytes); in _copy_from_iter_full()
800 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_nocache() argument
803 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter_nocache()
807 iterate_and_advance(i, bytes, v, in _copy_from_iter_nocache()
834 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_flushcache() argument
837 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter_flushcache()
841 iterate_and_advance(i, bytes, v, in _copy_from_iter_flushcache()
855 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_full_nocache() argument
858 if (unlikely(iov_iter_is_pipe(i))) { in _copy_from_iter_full_nocache()
862 if (unlikely(i->count < bytes)) in _copy_from_iter_full_nocache()
864 iterate_all_kinds(i, bytes, v, ({ in _copy_from_iter_full_nocache()
874 iov_iter_advance(i, bytes); in _copy_from_iter_full_nocache()
904 struct iov_iter *i) in copy_page_to_iter() argument
908 if (i->type & (ITER_BVEC|ITER_KVEC)) { in copy_page_to_iter()
910 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); in copy_page_to_iter()
913 } else if (unlikely(iov_iter_is_discard(i))) { in copy_page_to_iter()
914 if (unlikely(i->count < bytes)) in copy_page_to_iter()
915 bytes = i->count; in copy_page_to_iter()
916 i->count -= bytes; in copy_page_to_iter()
918 } else if (likely(!iov_iter_is_pipe(i))) in copy_page_to_iter()
919 return copy_page_to_iter_iovec(page, offset, bytes, i); in copy_page_to_iter()
921 return copy_page_to_iter_pipe(page, offset, bytes, i); in copy_page_to_iter()
926 struct iov_iter *i) in copy_page_from_iter() argument
930 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in copy_page_from_iter()
934 if (i->type & (ITER_BVEC|ITER_KVEC)) { in copy_page_from_iter()
936 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); in copy_page_from_iter()
940 return copy_page_from_iter_iovec(page, offset, bytes, i); in copy_page_from_iter()
944 static size_t pipe_zero(size_t bytes, struct iov_iter *i) in pipe_zero() argument
946 struct pipe_inode_info *pipe = i->pipe; in pipe_zero()
951 if (!sanity(i)) in pipe_zero()
954 bytes = n = push_pipe(i, bytes, &i_head, &off); in pipe_zero()
961 i->head = i_head; in pipe_zero()
962 i->iov_offset = off + chunk; in pipe_zero()
967 i->count -= bytes; in pipe_zero()
971 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) in iov_iter_zero() argument
973 if (unlikely(iov_iter_is_pipe(i))) in iov_iter_zero()
974 return pipe_zero(bytes, i); in iov_iter_zero()
975 iterate_and_advance(i, bytes, v, in iov_iter_zero()
986 struct iov_iter *i, unsigned long offset, size_t bytes) in iov_iter_copy_from_user_atomic() argument
993 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in iov_iter_copy_from_user_atomic()
998 iterate_all_kinds(i, bytes, v, in iov_iter_copy_from_user_atomic()
1009 static inline void pipe_truncate(struct iov_iter *i) in pipe_truncate() argument
1011 struct pipe_inode_info *pipe = i->pipe; in pipe_truncate()
1018 unsigned int i_head = i->head; in pipe_truncate()
1019 size_t off = i->iov_offset; in pipe_truncate()
1035 static void pipe_advance(struct iov_iter *i, size_t size) in pipe_advance() argument
1037 struct pipe_inode_info *pipe = i->pipe; in pipe_advance()
1038 if (unlikely(i->count < size)) in pipe_advance()
1039 size = i->count; in pipe_advance()
1043 unsigned int i_head = i->head; in pipe_advance()
1044 size_t off = i->iov_offset, left = size; in pipe_advance()
1055 i->head = i_head; in pipe_advance()
1056 i->iov_offset = buf->offset + left; in pipe_advance()
1058 i->count -= size; in pipe_advance()
1060 pipe_truncate(i); in pipe_advance()
1063 void iov_iter_advance(struct iov_iter *i, size_t size) in iov_iter_advance() argument
1065 if (unlikely(iov_iter_is_pipe(i))) { in iov_iter_advance()
1066 pipe_advance(i, size); in iov_iter_advance()
1069 if (unlikely(iov_iter_is_discard(i))) { in iov_iter_advance()
1070 i->count -= size; in iov_iter_advance()
1073 iterate_and_advance(i, size, v, 0, 0, 0) in iov_iter_advance()
1077 void iov_iter_revert(struct iov_iter *i, size_t unroll) in iov_iter_revert() argument
1083 i->count += unroll; in iov_iter_revert()
1084 if (unlikely(iov_iter_is_pipe(i))) { in iov_iter_revert()
1085 struct pipe_inode_info *pipe = i->pipe; in iov_iter_revert()
1087 unsigned int i_head = i->head; in iov_iter_revert()
1088 size_t off = i->iov_offset; in iov_iter_revert()
1097 if (!unroll && i_head == i->start_head) { in iov_iter_revert()
1105 i->iov_offset = off; in iov_iter_revert()
1106 i->head = i_head; in iov_iter_revert()
1107 pipe_truncate(i); in iov_iter_revert()
1110 if (unlikely(iov_iter_is_discard(i))) in iov_iter_revert()
1112 if (unroll <= i->iov_offset) { in iov_iter_revert()
1113 i->iov_offset -= unroll; in iov_iter_revert()
1116 unroll -= i->iov_offset; in iov_iter_revert()
1117 if (iov_iter_is_bvec(i)) { in iov_iter_revert()
1118 const struct bio_vec *bvec = i->bvec; in iov_iter_revert()
1121 i->nr_segs++; in iov_iter_revert()
1123 i->bvec = bvec; in iov_iter_revert()
1124 i->iov_offset = n - unroll; in iov_iter_revert()
1130 const struct iovec *iov = i->iov; in iov_iter_revert()
1133 i->nr_segs++; in iov_iter_revert()
1135 i->iov = iov; in iov_iter_revert()
1136 i->iov_offset = n - unroll; in iov_iter_revert()
1148 size_t iov_iter_single_seg_count(const struct iov_iter *i) in iov_iter_single_seg_count() argument
1150 if (unlikely(iov_iter_is_pipe(i))) in iov_iter_single_seg_count()
1151 return i->count; // it is a silly place, anyway in iov_iter_single_seg_count()
1152 if (i->nr_segs == 1) in iov_iter_single_seg_count()
1153 return i->count; in iov_iter_single_seg_count()
1154 if (unlikely(iov_iter_is_discard(i))) in iov_iter_single_seg_count()
1155 return i->count; in iov_iter_single_seg_count()
1156 else if (iov_iter_is_bvec(i)) in iov_iter_single_seg_count()
1157 return min(i->count, i->bvec->bv_len - i->iov_offset); in iov_iter_single_seg_count()
1159 return min(i->count, i->iov->iov_len - i->iov_offset); in iov_iter_single_seg_count()
1163 void iov_iter_kvec(struct iov_iter *i, unsigned int direction, in iov_iter_kvec() argument
1168 i->type = ITER_KVEC | (direction & (READ | WRITE)); in iov_iter_kvec()
1169 i->kvec = kvec; in iov_iter_kvec()
1170 i->nr_segs = nr_segs; in iov_iter_kvec()
1171 i->iov_offset = 0; in iov_iter_kvec()
1172 i->count = count; in iov_iter_kvec()
1176 void iov_iter_bvec(struct iov_iter *i, unsigned int direction, in iov_iter_bvec() argument
1181 i->type = ITER_BVEC | (direction & (READ | WRITE)); in iov_iter_bvec()
1182 i->bvec = bvec; in iov_iter_bvec()
1183 i->nr_segs = nr_segs; in iov_iter_bvec()
1184 i->iov_offset = 0; in iov_iter_bvec()
1185 i->count = count; in iov_iter_bvec()
1189 void iov_iter_pipe(struct iov_iter *i, unsigned int direction, in iov_iter_pipe() argument
1195 i->type = ITER_PIPE | READ; in iov_iter_pipe()
1196 i->pipe = pipe; in iov_iter_pipe()
1197 i->head = pipe->head; in iov_iter_pipe()
1198 i->iov_offset = 0; in iov_iter_pipe()
1199 i->count = count; in iov_iter_pipe()
1200 i->start_head = i->head; in iov_iter_pipe()
1205 * iov_iter_discard - Initialise an I/O iterator that discards data
1206 * @i: The iterator to initialise.
1208 * @count: The size of the I/O buffer in bytes.
1210 * Set up an I/O iterator that just discards everything that's written to it.
1213 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) in iov_iter_discard() argument
1216 i->type = ITER_DISCARD | READ; in iov_iter_discard()
1217 i->count = count; in iov_iter_discard()
1218 i->iov_offset = 0; in iov_iter_discard()
1222 unsigned long iov_iter_alignment(const struct iov_iter *i) in iov_iter_alignment() argument
1225 size_t size = i->count; in iov_iter_alignment()
1227 if (unlikely(iov_iter_is_pipe(i))) { in iov_iter_alignment()
1228 unsigned int p_mask = i->pipe->ring_size - 1; in iov_iter_alignment()
1230 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) in iov_iter_alignment()
1231 return size | i->iov_offset; in iov_iter_alignment()
1234 iterate_all_kinds(i, size, v, in iov_iter_alignment()
1243 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) in iov_iter_gap_alignment() argument
1246 size_t size = i->count; in iov_iter_gap_alignment()
1248 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in iov_iter_gap_alignment()
1253 iterate_all_kinds(i, size, v, in iov_iter_gap_alignment()
1265 static inline ssize_t __pipe_get_pages(struct iov_iter *i, in __pipe_get_pages() argument
1271 struct pipe_inode_info *pipe = i->pipe; in __pipe_get_pages()
1273 ssize_t n = push_pipe(i, maxsize, &iter_head, start); in __pipe_get_pages()
1288 static ssize_t pipe_get_pages(struct iov_iter *i, in pipe_get_pages() argument
1298 if (!sanity(i)) in pipe_get_pages()
1301 data_start(i, &iter_head, start); in pipe_get_pages()
1303 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); in pipe_get_pages()
1306 return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start); in pipe_get_pages()
1309 ssize_t iov_iter_get_pages(struct iov_iter *i, in iov_iter_get_pages() argument
1313 if (maxsize > i->count) in iov_iter_get_pages()
1314 maxsize = i->count; in iov_iter_get_pages()
1316 if (unlikely(iov_iter_is_pipe(i))) in iov_iter_get_pages()
1317 return pipe_get_pages(i, pages, maxsize, maxpages, start); in iov_iter_get_pages()
1318 if (unlikely(iov_iter_is_discard(i))) in iov_iter_get_pages()
1321 iterate_all_kinds(i, maxsize, v, ({ in iov_iter_get_pages()
1332 iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, in iov_iter_get_pages()
1355 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, in pipe_get_pages_alloc() argument
1366 if (!sanity(i)) in pipe_get_pages_alloc()
1369 data_start(i, &iter_head, start); in pipe_get_pages_alloc()
1371 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); in pipe_get_pages_alloc()
1380 n = __pipe_get_pages(i, maxsize, p, iter_head, start); in pipe_get_pages_alloc()
1388 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, in iov_iter_get_pages_alloc() argument
1394 if (maxsize > i->count) in iov_iter_get_pages_alloc()
1395 maxsize = i->count; in iov_iter_get_pages_alloc()
1397 if (unlikely(iov_iter_is_pipe(i))) in iov_iter_get_pages_alloc()
1398 return pipe_get_pages_alloc(i, pages, maxsize, start); in iov_iter_get_pages_alloc()
1399 if (unlikely(iov_iter_is_discard(i))) in iov_iter_get_pages_alloc()
1402 iterate_all_kinds(i, maxsize, v, ({ in iov_iter_get_pages_alloc()
1414 iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p); in iov_iter_get_pages_alloc()
1439 struct iov_iter *i) in csum_and_copy_from_iter() argument
1445 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in csum_and_copy_from_iter()
1449 iterate_and_advance(i, bytes, v, ({ in csum_and_copy_from_iter()
1478 struct iov_iter *i) in csum_and_copy_from_iter_full() argument
1484 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { in csum_and_copy_from_iter_full()
1488 if (unlikely(i->count < bytes)) in csum_and_copy_from_iter_full()
1490 iterate_all_kinds(i, bytes, v, ({ in csum_and_copy_from_iter_full()
1514 iov_iter_advance(i, bytes); in csum_and_copy_from_iter_full()
1520 struct iov_iter *i) in csum_and_copy_to_iter() argument
1527 if (unlikely(iov_iter_is_pipe(i))) in csum_and_copy_to_iter()
1528 return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i); in csum_and_copy_to_iter()
1532 if (unlikely(iov_iter_is_discard(i))) { in csum_and_copy_to_iter()
1536 iterate_and_advance(i, bytes, v, ({ in csum_and_copy_to_iter()
1566 struct iov_iter *i) in hash_and_copy_to_iter() argument
1573 copied = copy_to_iter(addr, bytes, i); in hash_and_copy_to_iter()
1584 int iov_iter_npages(const struct iov_iter *i, int maxpages) in iov_iter_npages() argument
1586 size_t size = i->count; in iov_iter_npages()
1591 if (unlikely(iov_iter_is_discard(i))) in iov_iter_npages()
1594 if (unlikely(iov_iter_is_pipe(i))) { in iov_iter_npages()
1595 struct pipe_inode_info *pipe = i->pipe; in iov_iter_npages()
1599 if (!sanity(i)) in iov_iter_npages()
1602 data_start(i, &iter_head, &off); in iov_iter_npages()
1607 } else iterate_all_kinds(i, size, v, ({ in iov_iter_npages()
1655 int ret = -EFAULT, i; in copy_compat_iovec_from_user() local
1660 for (i = 0; i < nr_segs; i++) { in copy_compat_iovec_from_user()
1664 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); in copy_compat_iovec_from_user()
1665 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); in copy_compat_iovec_from_user()
1672 iov[i].iov_base = compat_ptr(buf); in copy_compat_iovec_from_user()
1673 iov[i].iov_len = len; in copy_compat_iovec_from_user()
1734 struct iov_iter *i, bool compat) in __import_iovec() argument
1771 iov_iter_init(i, type, iov, nr_segs, total_len); in __import_iovec()
1790 * @i: Pointer to iterator that will be initialized on success.
1803 struct iovec **iovp, struct iov_iter *i) in import_iovec() argument
1805 return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, in import_iovec()
1811 struct iovec *iov, struct iov_iter *i) in import_single_range() argument
1820 iov_iter_init(i, rw, iov, 1, len); in import_single_range()
1829 * @i: &struct iov_iter to restore
1832 * Used after iov_iter_save_state() to bring restore @i, if operations may
1837 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) in iov_iter_restore() argument
1839 if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && in iov_iter_restore()
1840 !iov_iter_is_kvec(i)) in iov_iter_restore()
1842 i->iov_offset = state->iov_offset; in iov_iter_restore()
1843 i->count = state->count; in iov_iter_restore()
1854 if (iov_iter_is_bvec(i)) in iov_iter_restore()
1855 i->bvec -= state->nr_segs - i->nr_segs; in iov_iter_restore()
1857 i->iov -= state->nr_segs - i->nr_segs; in iov_iter_restore()
1858 i->nr_segs = state->nr_segs; in iov_iter_restore()