Lines Matching refs:i
11 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ argument
14 __p = i->iov; \
39 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ argument
41 __p = i->kvec; \
62 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ argument
67 for_each_bvec(__v, i->bvec, __bi, __start) { \
74 #define iterate_all_kinds(i, n, v, I, B, K) { \ argument
75 size_t skip = i->iov_offset; \
76 if (unlikely(i->type & ITER_BVEC)) { \
79 iterate_bvec(i, n, v, __bi, skip, (B)) \
80 } else if (unlikely(i->type & ITER_KVEC)) { \
83 iterate_kvec(i, n, v, kvec, skip, (K)) \
87 iterate_iovec(i, n, v, iov, skip, (I)) \
91 #define iterate_and_advance(i, n, v, I, B, K) { \ argument
92 if (unlikely(i->count < n)) \
93 n = i->count; \
94 if (i->count) { \
95 size_t skip = i->iov_offset; \
96 if (unlikely(i->type & ITER_BVEC)) { \
97 const struct bio_vec *bvec = i->bvec; \
100 iterate_bvec(i, n, v, __bi, skip, (B)) \
101 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
102 i->nr_segs -= i->bvec - bvec; \
104 } else if (unlikely(i->type & ITER_KVEC)) { \
107 iterate_kvec(i, n, v, kvec, skip, (K)) \
112 i->nr_segs -= kvec - i->kvec; \
113 i->kvec = kvec; \
117 iterate_iovec(i, n, v, iov, skip, (I)) \
122 i->nr_segs -= iov - i->iov; \
123 i->iov = iov; \
125 i->count -= n; \
126 i->iov_offset = skip; \
131 struct iov_iter *i) in copy_page_to_iter_iovec() argument
138 if (unlikely(bytes > i->count)) in copy_page_to_iter_iovec()
139 bytes = i->count; in copy_page_to_iter_iovec()
145 iov = i->iov; in copy_page_to_iter_iovec()
146 skip = i->iov_offset; in copy_page_to_iter_iovec()
206 i->count -= wanted - bytes; in copy_page_to_iter_iovec()
207 i->nr_segs -= iov - i->iov; in copy_page_to_iter_iovec()
208 i->iov = iov; in copy_page_to_iter_iovec()
209 i->iov_offset = skip; in copy_page_to_iter_iovec()
214 struct iov_iter *i) in copy_page_from_iter_iovec() argument
221 if (unlikely(bytes > i->count)) in copy_page_from_iter_iovec()
222 bytes = i->count; in copy_page_from_iter_iovec()
228 iov = i->iov; in copy_page_from_iter_iovec()
229 skip = i->iov_offset; in copy_page_from_iter_iovec()
289 i->count -= wanted - bytes; in copy_page_from_iter_iovec()
290 i->nr_segs -= iov - i->iov; in copy_page_from_iter_iovec()
291 i->iov = iov; in copy_page_from_iter_iovec()
292 i->iov_offset = skip; in copy_page_from_iter_iovec()
297 static bool sanity(const struct iov_iter *i) in sanity() argument
299 struct pipe_inode_info *pipe = i->pipe; in sanity()
300 int idx = i->idx; in sanity()
302 if (i->iov_offset) { in sanity()
310 if (unlikely(p->offset + p->len != i->iov_offset)) in sanity()
318 printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset); in sanity()
331 #define sanity(i) true argument
340 struct iov_iter *i) in copy_page_to_iter_pipe() argument
342 struct pipe_inode_info *pipe = i->pipe; in copy_page_to_iter_pipe()
347 if (unlikely(bytes > i->count)) in copy_page_to_iter_pipe()
348 bytes = i->count; in copy_page_to_iter_pipe()
353 if (!sanity(i)) in copy_page_to_iter_pipe()
356 off = i->iov_offset; in copy_page_to_iter_pipe()
357 idx = i->idx; in copy_page_to_iter_pipe()
363 i->iov_offset += bytes; in copy_page_to_iter_pipe()
376 i->iov_offset = offset + bytes; in copy_page_to_iter_pipe()
377 i->idx = idx; in copy_page_to_iter_pipe()
379 i->count -= bytes; in copy_page_to_iter_pipe()
390 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) in iov_iter_fault_in_readable() argument
392 size_t skip = i->iov_offset; in iov_iter_fault_in_readable()
397 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { in iov_iter_fault_in_readable()
398 iterate_iovec(i, bytes, v, iov, skip, ({ in iov_iter_fault_in_readable()
408 void iov_iter_init(struct iov_iter *i, int direction, in iov_iter_init() argument
415 i->type = direction; in iov_iter_init()
416 i->kvec = (struct kvec *)iov; in iov_iter_init()
418 i->type = direction; in iov_iter_init()
419 i->iov = iov; in iov_iter_init()
421 i->nr_segs = nr_segs; in iov_iter_init()
422 i->iov_offset = 0; in iov_iter_init()
423 i->count = count; in iov_iter_init()
453 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp) in data_start() argument
455 size_t off = i->iov_offset; in data_start()
456 int idx = i->idx; in data_start()
457 if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) { in data_start()
458 idx = next_idx(idx, i->pipe); in data_start()
465 static size_t push_pipe(struct iov_iter *i, size_t size, in push_pipe() argument
468 struct pipe_inode_info *pipe = i->pipe; in push_pipe()
473 if (unlikely(size > i->count)) in push_pipe()
474 size = i->count; in push_pipe()
479 data_start(i, &idx, &off); in push_pipe()
511 struct iov_iter *i) in copy_pipe_to_iter() argument
513 struct pipe_inode_info *pipe = i->pipe; in copy_pipe_to_iter()
517 if (!sanity(i)) in copy_pipe_to_iter()
520 bytes = n = push_pipe(i, bytes, &idx, &off); in copy_pipe_to_iter()
526 i->idx = idx; in copy_pipe_to_iter()
527 i->iov_offset = off + chunk; in copy_pipe_to_iter()
531 i->count -= bytes; in copy_pipe_to_iter()
535 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) in copy_to_iter() argument
538 if (unlikely(i->type & ITER_PIPE)) in copy_to_iter()
539 return copy_pipe_to_iter(addr, bytes, i); in copy_to_iter()
540 iterate_and_advance(i, bytes, v, in copy_to_iter()
552 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) in copy_from_iter() argument
555 if (unlikely(i->type & ITER_PIPE)) { in copy_from_iter()
559 iterate_and_advance(i, bytes, v, in copy_from_iter()
571 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) in copy_from_iter_nocache() argument
574 if (unlikely(i->type & ITER_PIPE)) { in copy_from_iter_nocache()
578 iterate_and_advance(i, bytes, v, in copy_from_iter_nocache()
591 struct iov_iter *i) in copy_page_to_iter() argument
593 if (i->type & (ITER_BVEC|ITER_KVEC)) { in copy_page_to_iter()
595 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); in copy_page_to_iter()
598 } else if (likely(!(i->type & ITER_PIPE))) in copy_page_to_iter()
599 return copy_page_to_iter_iovec(page, offset, bytes, i); in copy_page_to_iter()
601 return copy_page_to_iter_pipe(page, offset, bytes, i); in copy_page_to_iter()
606 struct iov_iter *i) in copy_page_from_iter() argument
608 if (unlikely(i->type & ITER_PIPE)) { in copy_page_from_iter()
612 if (i->type & (ITER_BVEC|ITER_KVEC)) { in copy_page_from_iter()
614 size_t wanted = copy_from_iter(kaddr + offset, bytes, i); in copy_page_from_iter()
618 return copy_page_from_iter_iovec(page, offset, bytes, i); in copy_page_from_iter()
622 static size_t pipe_zero(size_t bytes, struct iov_iter *i) in pipe_zero() argument
624 struct pipe_inode_info *pipe = i->pipe; in pipe_zero()
628 if (!sanity(i)) in pipe_zero()
631 bytes = n = push_pipe(i, bytes, &idx, &off); in pipe_zero()
638 i->idx = idx; in pipe_zero()
639 i->iov_offset = off + chunk; in pipe_zero()
642 i->count -= bytes; in pipe_zero()
646 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) in iov_iter_zero() argument
648 if (unlikely(i->type & ITER_PIPE)) in iov_iter_zero()
649 return pipe_zero(bytes, i); in iov_iter_zero()
650 iterate_and_advance(i, bytes, v, in iov_iter_zero()
661 struct iov_iter *i, unsigned long offset, size_t bytes) in iov_iter_copy_from_user_atomic() argument
664 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_copy_from_user_atomic()
669 iterate_all_kinds(i, bytes, v, in iov_iter_copy_from_user_atomic()
681 static inline void pipe_truncate(struct iov_iter *i) in pipe_truncate() argument
683 struct pipe_inode_info *pipe = i->pipe; in pipe_truncate()
685 size_t off = i->iov_offset; in pipe_truncate()
686 int idx = i->idx; in pipe_truncate()
701 static void pipe_advance(struct iov_iter *i, size_t size) in pipe_advance() argument
703 struct pipe_inode_info *pipe = i->pipe; in pipe_advance()
704 if (unlikely(i->count < size)) in pipe_advance()
705 size = i->count; in pipe_advance()
708 size_t off = i->iov_offset, left = size; in pipe_advance()
709 int idx = i->idx; in pipe_advance()
719 i->idx = idx; in pipe_advance()
720 i->iov_offset = buf->offset + left; in pipe_advance()
722 i->count -= size; in pipe_advance()
724 pipe_truncate(i); in pipe_advance()
727 void iov_iter_advance(struct iov_iter *i, size_t size) in iov_iter_advance() argument
729 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_advance()
730 pipe_advance(i, size); in iov_iter_advance()
733 iterate_and_advance(i, size, v, 0, 0, 0) in iov_iter_advance()
737 void iov_iter_revert(struct iov_iter *i, size_t unroll) in iov_iter_revert() argument
741 i->count += unroll; in iov_iter_revert()
742 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_revert()
743 struct pipe_inode_info *pipe = i->pipe; in iov_iter_revert()
744 int idx = i->idx; in iov_iter_revert()
745 size_t off = i->iov_offset; in iov_iter_revert()
753 if (!unroll && idx == i->start_idx) { in iov_iter_revert()
761 i->iov_offset = off; in iov_iter_revert()
762 i->idx = idx; in iov_iter_revert()
763 pipe_truncate(i); in iov_iter_revert()
766 if (unroll <= i->iov_offset) { in iov_iter_revert()
767 i->iov_offset -= unroll; in iov_iter_revert()
770 unroll -= i->iov_offset; in iov_iter_revert()
771 if (i->type & ITER_BVEC) { in iov_iter_revert()
772 const struct bio_vec *bvec = i->bvec; in iov_iter_revert()
775 i->nr_segs++; in iov_iter_revert()
777 i->bvec = bvec; in iov_iter_revert()
778 i->iov_offset = n - unroll; in iov_iter_revert()
784 const struct iovec *iov = i->iov; in iov_iter_revert()
787 i->nr_segs++; in iov_iter_revert()
789 i->iov = iov; in iov_iter_revert()
790 i->iov_offset = n - unroll; in iov_iter_revert()
802 size_t iov_iter_single_seg_count(const struct iov_iter *i) in iov_iter_single_seg_count() argument
804 if (unlikely(i->type & ITER_PIPE)) in iov_iter_single_seg_count()
805 return i->count; // it is a silly place, anyway in iov_iter_single_seg_count()
806 if (i->nr_segs == 1) in iov_iter_single_seg_count()
807 return i->count; in iov_iter_single_seg_count()
808 else if (i->type & ITER_BVEC) in iov_iter_single_seg_count()
809 return min(i->count, i->bvec->bv_len - i->iov_offset); in iov_iter_single_seg_count()
811 return min(i->count, i->iov->iov_len - i->iov_offset); in iov_iter_single_seg_count()
815 void iov_iter_kvec(struct iov_iter *i, int direction, in iov_iter_kvec() argument
820 i->type = direction; in iov_iter_kvec()
821 i->kvec = kvec; in iov_iter_kvec()
822 i->nr_segs = nr_segs; in iov_iter_kvec()
823 i->iov_offset = 0; in iov_iter_kvec()
824 i->count = count; in iov_iter_kvec()
828 void iov_iter_bvec(struct iov_iter *i, int direction, in iov_iter_bvec() argument
833 i->type = direction; in iov_iter_bvec()
834 i->bvec = bvec; in iov_iter_bvec()
835 i->nr_segs = nr_segs; in iov_iter_bvec()
836 i->iov_offset = 0; in iov_iter_bvec()
837 i->count = count; in iov_iter_bvec()
841 void iov_iter_pipe(struct iov_iter *i, int direction, in iov_iter_pipe() argument
847 i->type = direction; in iov_iter_pipe()
848 i->pipe = pipe; in iov_iter_pipe()
849 i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); in iov_iter_pipe()
850 i->iov_offset = 0; in iov_iter_pipe()
851 i->count = count; in iov_iter_pipe()
852 i->start_idx = i->idx; in iov_iter_pipe()
856 unsigned long iov_iter_alignment(const struct iov_iter *i) in iov_iter_alignment() argument
859 size_t size = i->count; in iov_iter_alignment()
864 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_alignment()
865 if (i->iov_offset && allocated(&i->pipe->bufs[i->idx])) in iov_iter_alignment()
866 return size | i->iov_offset; in iov_iter_alignment()
869 iterate_all_kinds(i, size, v, in iov_iter_alignment()
878 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) in iov_iter_gap_alignment() argument
881 size_t size = i->count; in iov_iter_gap_alignment()
885 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_gap_alignment()
890 iterate_all_kinds(i, size, v, in iov_iter_gap_alignment()
902 static inline size_t __pipe_get_pages(struct iov_iter *i, in __pipe_get_pages() argument
908 struct pipe_inode_info *pipe = i->pipe; in __pipe_get_pages()
909 ssize_t n = push_pipe(i, maxsize, &idx, start); in __pipe_get_pages()
924 static ssize_t pipe_get_pages(struct iov_iter *i, in pipe_get_pages() argument
932 if (!sanity(i)) in pipe_get_pages()
935 data_start(i, &idx, start); in pipe_get_pages()
937 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; in pipe_get_pages()
940 return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start); in pipe_get_pages()
943 ssize_t iov_iter_get_pages(struct iov_iter *i, in iov_iter_get_pages() argument
947 if (maxsize > i->count) in iov_iter_get_pages()
948 maxsize = i->count; in iov_iter_get_pages()
953 if (unlikely(i->type & ITER_PIPE)) in iov_iter_get_pages()
954 return pipe_get_pages(i, pages, maxsize, maxpages, start); in iov_iter_get_pages()
955 iterate_all_kinds(i, maxsize, v, ({ in iov_iter_get_pages()
965 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); in iov_iter_get_pages()
990 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, in pipe_get_pages_alloc() argument
999 if (!sanity(i)) in pipe_get_pages_alloc()
1002 data_start(i, &idx, start); in pipe_get_pages_alloc()
1004 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; in pipe_get_pages_alloc()
1013 n = __pipe_get_pages(i, maxsize, p, idx, start); in pipe_get_pages_alloc()
1021 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, in iov_iter_get_pages_alloc() argument
1027 if (maxsize > i->count) in iov_iter_get_pages_alloc()
1028 maxsize = i->count; in iov_iter_get_pages_alloc()
1033 if (unlikely(i->type & ITER_PIPE)) in iov_iter_get_pages_alloc()
1034 return pipe_get_pages_alloc(i, pages, maxsize, start); in iov_iter_get_pages_alloc()
1035 iterate_all_kinds(i, maxsize, v, ({ in iov_iter_get_pages_alloc()
1046 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); in iov_iter_get_pages_alloc()
1070 struct iov_iter *i) in csum_and_copy_from_iter() argument
1076 if (unlikely(i->type & ITER_PIPE)) { in csum_and_copy_from_iter()
1080 iterate_and_advance(i, bytes, v, ({ in csum_and_copy_from_iter()
1112 struct iov_iter *i) in csum_and_copy_to_iter() argument
1118 if (unlikely(i->type & ITER_PIPE)) { in csum_and_copy_to_iter()
1122 iterate_and_advance(i, bytes, v, ({ in csum_and_copy_to_iter()
1153 int iov_iter_npages(const struct iov_iter *i, int maxpages) in iov_iter_npages() argument
1155 size_t size = i->count; in iov_iter_npages()
1161 if (unlikely(i->type & ITER_PIPE)) { in iov_iter_npages()
1162 struct pipe_inode_info *pipe = i->pipe; in iov_iter_npages()
1166 if (!sanity(i)) in iov_iter_npages()
1169 data_start(i, &idx, &off); in iov_iter_npages()
1174 } else iterate_all_kinds(i, size, v, ({ in iov_iter_npages()
1239 struct iovec **iov, struct iov_iter *i) in import_iovec() argument
1251 iov_iter_init(i, type, p, nr_segs, n); in import_iovec()
1262 struct iovec **iov, struct iov_iter *i) in compat_import_iovec() argument
1274 iov_iter_init(i, type, p, nr_segs, n); in compat_import_iovec()
1281 struct iovec *iov, struct iov_iter *i) in import_single_range() argument
1290 iov_iter_init(i, rw, iov, 1, len); in import_single_range()