Lines Matching refs:i
12 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ argument
15 __p = i->iov; \
40 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ argument
42 __p = i->kvec; \
63 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ argument
68 for_each_bvec(__v, i->bvec, __bi, __start) { \
75 #define iterate_all_kinds(i, n, v, I, B, K) { \ argument
77 size_t skip = i->iov_offset; \
78 if (unlikely(i->type & ITER_BVEC)) { \
81 iterate_bvec(i, n, v, __bi, skip, (B)) \
82 } else if (unlikely(i->type & ITER_KVEC)) { \
85 iterate_kvec(i, n, v, kvec, skip, (K)) \
89 iterate_iovec(i, n, v, iov, skip, (I)) \
94 #define iterate_and_advance(i, n, v, I, B, K) { \ argument
95 if (unlikely(i->count < n)) \
96 n = i->count; \
97 if (i->count) { \
98 size_t skip = i->iov_offset; \
99 if (unlikely(i->type & ITER_BVEC)) { \
100 const struct bio_vec *bvec = i->bvec; \
103 iterate_bvec(i, n, v, __bi, skip, (B)) \
104 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
105 i->nr_segs -= i->bvec - bvec; \
107 } else if (unlikely(i->type & ITER_KVEC)) { \
110 iterate_kvec(i, n, v, kvec, skip, (K)) \
115 i->nr_segs -= kvec - i->kvec; \
116 i->kvec = kvec; \
120 iterate_iovec(i, n, v, iov, skip, (I)) \
125 i->nr_segs -= iov - i->iov; \
126 i->iov = iov; \
128 i->count -= n; \
129 i->iov_offset = skip; \
152 struct iov_iter *i) in copy_page_to_iter_iovec() argument
159 if (unlikely(bytes > i->count)) in copy_page_to_iter_iovec()
160 bytes = i->count; in copy_page_to_iter_iovec()
167 iov = i->iov; in copy_page_to_iter_iovec()
168 skip = i->iov_offset; in copy_page_to_iter_iovec()
228 i->count -= wanted - bytes; in copy_page_to_iter_iovec()
229 i->nr_segs -= iov - i->iov; in copy_page_to_iter_iovec()
230 i->iov = iov; in copy_page_to_iter_iovec()
231 i->iov_offset = skip; in copy_page_to_iter_iovec()
236 struct iov_iter *i) in copy_page_from_iter_iovec() argument
243 if (unlikely(bytes > i->count)) in copy_page_from_iter_iovec()
244 bytes = i->count; in copy_page_from_iter_iovec()
251 iov = i->iov; in copy_page_from_iter_iovec()
252 skip = i->iov_offset; in copy_page_from_iter_iovec()
312 i->count -= wanted - bytes; in copy_page_from_iter_iovec()
313 i->nr_segs -= iov - i->iov; in copy_page_from_iter_iovec()
314 i->iov = iov; in copy_page_from_iter_iovec()
315 i->iov_offset = skip; in copy_page_from_iter_iovec()
320 static bool sanity(const struct iov_iter *i) in sanity() argument
322 struct pipe_inode_info *pipe = i->pipe; in sanity()
323 int idx = i->idx; in sanity()
325 if (i->iov_offset) { in sanity()
333 if (unlikely(p->offset + p->len != i->iov_offset)) in sanity()
341 printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset); in sanity()
354 #define sanity(i) true argument
363 struct iov_iter *i) in copy_page_to_iter_pipe() argument
365 struct pipe_inode_info *pipe = i->pipe; in copy_page_to_iter_pipe()
370 if (unlikely(bytes > i->count)) in copy_page_to_iter_pipe()
371 bytes = i->count; in copy_page_to_iter_pipe()
376 if (!sanity(i)) in copy_page_to_iter_pipe()
379 off = i->iov_offset; in copy_page_to_iter_pipe()
380 idx = i->idx; in copy_page_to_iter_pipe()
386 i->iov_offset += bytes; in copy_page_to_iter_pipe()
399 i->iov_offset = offset + bytes; in copy_page_to_iter_pipe()
400 i->idx = idx; in copy_page_to_iter_pipe()
402 i->count -= bytes; in copy_page_to_iter_pipe()
413 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) in iov_iter_fault_in_readable() argument
415 size_t skip = i->iov_offset; in iov_iter_fault_in_readable()
420 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { in iov_iter_fault_in_readable()
421 iterate_iovec(i, bytes, v, iov, skip, ({ in iov_iter_fault_in_readable()
431 void iov_iter_init(struct iov_iter *i, int direction, in iov_iter_init() argument
438 i->type = direction; in iov_iter_init()
439 i->kvec = (struct kvec *)iov; in iov_iter_init()
441 i->type = direction; in iov_iter_init()
442 i->iov = iov; in iov_iter_init()
444 i->nr_segs = nr_segs; in iov_iter_init()
445 i->iov_offset = 0; in iov_iter_init()
446 i->count = count; in iov_iter_init()
476 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp) in data_start() argument
478 size_t off = i->iov_offset; in data_start()
479 int idx = i->idx; in data_start()
480 if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) { in data_start()
481 idx = next_idx(idx, i->pipe); in data_start()
488 static size_t push_pipe(struct iov_iter *i, size_t size, in push_pipe() argument
491 struct pipe_inode_info *pipe = i->pipe; in push_pipe()
496 if (unlikely(size > i->count)) in push_pipe()
497 size = i->count; in push_pipe()
502 data_start(i, &idx, &off); in push_pipe()
534 struct iov_iter *i) in copy_pipe_to_iter() argument
536 struct pipe_inode_info *pipe = i->pipe; in copy_pipe_to_iter()
540 if (!sanity(i)) in copy_pipe_to_iter()
543 bytes = n = push_pipe(i, bytes, &idx, &off); in copy_pipe_to_iter()
549 i->idx = idx; in copy_pipe_to_iter()
550 i->iov_offset = off + chunk; in copy_pipe_to_iter()
554 i->count -= bytes; in copy_pipe_to_iter()
558 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in _copy_to_iter() argument
561 if (unlikely(i->type & ITER_PIPE)) in _copy_to_iter()
562 return copy_pipe_to_iter(addr, bytes, i); in _copy_to_iter()
563 if (iter_is_iovec(i)) in _copy_to_iter()
565 iterate_and_advance(i, bytes, v, in _copy_to_iter()
576 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter() argument
579 if (unlikely(i->type & ITER_PIPE)) { in _copy_from_iter()
583 if (iter_is_iovec(i)) in _copy_from_iter()
585 iterate_and_advance(i, bytes, v, in _copy_from_iter()
596 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_full() argument
599 if (unlikely(i->type & ITER_PIPE)) { in _copy_from_iter_full()
603 if (unlikely(i->count < bytes)) in _copy_from_iter_full()
606 if (iter_is_iovec(i)) in _copy_from_iter_full()
608 iterate_all_kinds(i, bytes, v, ({ in _copy_from_iter_full()
618 iov_iter_advance(i, bytes); in _copy_from_iter_full()
623 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_nocache() argument
626 if (unlikely(i->type & ITER_PIPE)) { in _copy_from_iter_nocache()
630 iterate_and_advance(i, bytes, v, in _copy_from_iter_nocache()
643 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_flushcache() argument
646 if (unlikely(i->type & ITER_PIPE)) { in _copy_from_iter_flushcache()
650 iterate_and_advance(i, bytes, v, in _copy_from_iter_flushcache()
664 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) in _copy_from_iter_full_nocache() argument
667 if (unlikely(i->type & ITER_PIPE)) { in _copy_from_iter_full_nocache()
671 if (unlikely(i->count < bytes)) in _copy_from_iter_full_nocache()
673 iterate_all_kinds(i, bytes, v, ({ in _copy_from_iter_full_nocache()
683 iov_iter_advance(i, bytes); in _copy_from_iter_full_nocache()
713 struct iov_iter *i) in copy_page_to_iter() argument
717 if (i->type & (ITER_BVEC|ITER_KVEC)) { in copy_page_to_iter()
719 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); in copy_page_to_iter()
722 } else if (likely(!(i->type & ITER_PIPE))) in copy_page_to_iter()
723 return copy_page_to_iter_iovec(page, offset, bytes, i); in copy_page_to_iter()
725 return copy_page_to_iter_pipe(page, offset, bytes, i); in copy_page_to_iter()
730 struct iov_iter *i) in copy_page_from_iter() argument
734 if (unlikely(i->type & ITER_PIPE)) { in copy_page_from_iter()
738 if (i->type & (ITER_BVEC|ITER_KVEC)) { in copy_page_from_iter()
740 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); in copy_page_from_iter()
744 return copy_page_from_iter_iovec(page, offset, bytes, i); in copy_page_from_iter()
748 static size_t pipe_zero(size_t bytes, struct iov_iter *i) in pipe_zero() argument
750 struct pipe_inode_info *pipe = i->pipe; in pipe_zero()
754 if (!sanity(i)) in pipe_zero()
757 bytes = n = push_pipe(i, bytes, &idx, &off); in pipe_zero()
764 i->idx = idx; in pipe_zero()
765 i->iov_offset = off + chunk; in pipe_zero()
768 i->count -= bytes; in pipe_zero()
772 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) in iov_iter_zero() argument
774 if (unlikely(i->type & ITER_PIPE)) in iov_iter_zero()
775 return pipe_zero(bytes, i); in iov_iter_zero()
776 iterate_and_advance(i, bytes, v, in iov_iter_zero()
787 struct iov_iter *i, unsigned long offset, size_t bytes) in iov_iter_copy_from_user_atomic() argument
794 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_copy_from_user_atomic()
799 iterate_all_kinds(i, bytes, v, in iov_iter_copy_from_user_atomic()
810 static inline void pipe_truncate(struct iov_iter *i) in pipe_truncate() argument
812 struct pipe_inode_info *pipe = i->pipe; in pipe_truncate()
814 size_t off = i->iov_offset; in pipe_truncate()
815 int idx = i->idx; in pipe_truncate()
830 static void pipe_advance(struct iov_iter *i, size_t size) in pipe_advance() argument
832 struct pipe_inode_info *pipe = i->pipe; in pipe_advance()
833 if (unlikely(i->count < size)) in pipe_advance()
834 size = i->count; in pipe_advance()
837 size_t off = i->iov_offset, left = size; in pipe_advance()
838 int idx = i->idx; in pipe_advance()
848 i->idx = idx; in pipe_advance()
849 i->iov_offset = buf->offset + left; in pipe_advance()
851 i->count -= size; in pipe_advance()
853 pipe_truncate(i); in pipe_advance()
856 void iov_iter_advance(struct iov_iter *i, size_t size) in iov_iter_advance() argument
858 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_advance()
859 pipe_advance(i, size); in iov_iter_advance()
862 iterate_and_advance(i, size, v, 0, 0, 0) in iov_iter_advance()
866 void iov_iter_revert(struct iov_iter *i, size_t unroll) in iov_iter_revert() argument
872 i->count += unroll; in iov_iter_revert()
873 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_revert()
874 struct pipe_inode_info *pipe = i->pipe; in iov_iter_revert()
875 int idx = i->idx; in iov_iter_revert()
876 size_t off = i->iov_offset; in iov_iter_revert()
884 if (!unroll && idx == i->start_idx) { in iov_iter_revert()
892 i->iov_offset = off; in iov_iter_revert()
893 i->idx = idx; in iov_iter_revert()
894 pipe_truncate(i); in iov_iter_revert()
897 if (unroll <= i->iov_offset) { in iov_iter_revert()
898 i->iov_offset -= unroll; in iov_iter_revert()
901 unroll -= i->iov_offset; in iov_iter_revert()
902 if (i->type & ITER_BVEC) { in iov_iter_revert()
903 const struct bio_vec *bvec = i->bvec; in iov_iter_revert()
906 i->nr_segs++; in iov_iter_revert()
908 i->bvec = bvec; in iov_iter_revert()
909 i->iov_offset = n - unroll; in iov_iter_revert()
915 const struct iovec *iov = i->iov; in iov_iter_revert()
918 i->nr_segs++; in iov_iter_revert()
920 i->iov = iov; in iov_iter_revert()
921 i->iov_offset = n - unroll; in iov_iter_revert()
933 size_t iov_iter_single_seg_count(const struct iov_iter *i) in iov_iter_single_seg_count() argument
935 if (unlikely(i->type & ITER_PIPE)) in iov_iter_single_seg_count()
936 return i->count; // it is a silly place, anyway in iov_iter_single_seg_count()
937 if (i->nr_segs == 1) in iov_iter_single_seg_count()
938 return i->count; in iov_iter_single_seg_count()
939 else if (i->type & ITER_BVEC) in iov_iter_single_seg_count()
940 return min(i->count, i->bvec->bv_len - i->iov_offset); in iov_iter_single_seg_count()
942 return min(i->count, i->iov->iov_len - i->iov_offset); in iov_iter_single_seg_count()
946 void iov_iter_kvec(struct iov_iter *i, int direction, in iov_iter_kvec() argument
951 i->type = direction; in iov_iter_kvec()
952 i->kvec = kvec; in iov_iter_kvec()
953 i->nr_segs = nr_segs; in iov_iter_kvec()
954 i->iov_offset = 0; in iov_iter_kvec()
955 i->count = count; in iov_iter_kvec()
959 void iov_iter_bvec(struct iov_iter *i, int direction, in iov_iter_bvec() argument
964 i->type = direction; in iov_iter_bvec()
965 i->bvec = bvec; in iov_iter_bvec()
966 i->nr_segs = nr_segs; in iov_iter_bvec()
967 i->iov_offset = 0; in iov_iter_bvec()
968 i->count = count; in iov_iter_bvec()
972 void iov_iter_pipe(struct iov_iter *i, int direction, in iov_iter_pipe() argument
978 i->type = direction; in iov_iter_pipe()
979 i->pipe = pipe; in iov_iter_pipe()
980 i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); in iov_iter_pipe()
981 i->iov_offset = 0; in iov_iter_pipe()
982 i->count = count; in iov_iter_pipe()
983 i->start_idx = i->idx; in iov_iter_pipe()
987 unsigned long iov_iter_alignment(const struct iov_iter *i) in iov_iter_alignment() argument
990 size_t size = i->count; in iov_iter_alignment()
992 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_alignment()
993 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx])) in iov_iter_alignment()
994 return size | i->iov_offset; in iov_iter_alignment()
997 iterate_all_kinds(i, size, v, in iov_iter_alignment()
1006 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) in iov_iter_gap_alignment() argument
1009 size_t size = i->count; in iov_iter_gap_alignment()
1011 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_gap_alignment()
1016 iterate_all_kinds(i, size, v, in iov_iter_gap_alignment()
1028 static inline size_t __pipe_get_pages(struct iov_iter *i, in __pipe_get_pages() argument
1034 struct pipe_inode_info *pipe = i->pipe; in __pipe_get_pages()
1035 ssize_t n = push_pipe(i, maxsize, &idx, start); in __pipe_get_pages()
1050 static ssize_t pipe_get_pages(struct iov_iter *i, in pipe_get_pages() argument
1061 if (!sanity(i)) in pipe_get_pages()
1064 data_start(i, &idx, start); in pipe_get_pages()
1066 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; in pipe_get_pages()
1069 return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start); in pipe_get_pages()
1072 ssize_t iov_iter_get_pages(struct iov_iter *i, in iov_iter_get_pages() argument
1076 if (maxsize > i->count) in iov_iter_get_pages()
1077 maxsize = i->count; in iov_iter_get_pages()
1079 if (unlikely(i->type & ITER_PIPE)) in iov_iter_get_pages()
1080 return pipe_get_pages(i, pages, maxsize, maxpages, start); in iov_iter_get_pages()
1081 iterate_all_kinds(i, maxsize, v, ({ in iov_iter_get_pages()
1091 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); in iov_iter_get_pages()
1113 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, in pipe_get_pages_alloc() argument
1125 if (!sanity(i)) in pipe_get_pages_alloc()
1128 data_start(i, &idx, start); in pipe_get_pages_alloc()
1130 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; in pipe_get_pages_alloc()
1139 n = __pipe_get_pages(i, maxsize, p, idx, start); in pipe_get_pages_alloc()
1147 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, in iov_iter_get_pages_alloc() argument
1153 if (maxsize > i->count) in iov_iter_get_pages_alloc()
1154 maxsize = i->count; in iov_iter_get_pages_alloc()
1156 if (unlikely(i->type & ITER_PIPE)) in iov_iter_get_pages_alloc()
1157 return pipe_get_pages_alloc(i, pages, maxsize, start); in iov_iter_get_pages_alloc()
1158 iterate_all_kinds(i, maxsize, v, ({ in iov_iter_get_pages_alloc()
1169 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); in iov_iter_get_pages_alloc()
1193 struct iov_iter *i) in csum_and_copy_from_iter() argument
1199 if (unlikely(i->type & ITER_PIPE)) { in csum_and_copy_from_iter()
1203 iterate_and_advance(i, bytes, v, ({ in csum_and_copy_from_iter()
1235 struct iov_iter *i) in csum_and_copy_from_iter_full() argument
1241 if (unlikely(i->type & ITER_PIPE)) { in csum_and_copy_from_iter_full()
1245 if (unlikely(i->count < bytes)) in csum_and_copy_from_iter_full()
1247 iterate_all_kinds(i, bytes, v, ({ in csum_and_copy_from_iter_full()
1274 iov_iter_advance(i, bytes); in csum_and_copy_from_iter_full()
1280 struct iov_iter *i) in csum_and_copy_to_iter() argument
1286 if (unlikely(i->type & ITER_PIPE)) { in csum_and_copy_to_iter()
1290 iterate_and_advance(i, bytes, v, ({ in csum_and_copy_to_iter()
1321 int iov_iter_npages(const struct iov_iter *i, int maxpages) in iov_iter_npages() argument
1323 size_t size = i->count; in iov_iter_npages()
1329 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_npages()
1330 struct pipe_inode_info *pipe = i->pipe; in iov_iter_npages()
1334 if (!sanity(i)) in iov_iter_npages()
1337 data_start(i, &idx, &off); in iov_iter_npages()
1342 } else iterate_all_kinds(i, size, v, ({ in iov_iter_npages()
1407 struct iovec **iov, struct iov_iter *i) in import_iovec() argument
1419 iov_iter_init(i, type, p, nr_segs, n); in import_iovec()
1430 struct iovec **iov, struct iov_iter *i) in compat_import_iovec() argument
1442 iov_iter_init(i, type, p, nr_segs, n); in compat_import_iovec()
1449 struct iovec *iov, struct iov_iter *i) in import_single_range() argument
1458 iov_iter_init(i, rw, iov, 1, len); in import_single_range()