164 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
172 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
173 		bytes = i->count;  in copy_page_to_iter_iovec()
175 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
179 	wanted = bytes;  in copy_page_to_iter_iovec()
183 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
194 		bytes -= copy;  in copy_page_to_iter_iovec()
196 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
199 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
204 			bytes -= copy;  in copy_page_to_iter_iovec()
206 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
213 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
223 	bytes -= copy;  in copy_page_to_iter_iovec()
224 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
227 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
232 		bytes -= copy;  in copy_page_to_iter_iovec()
241 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
245 	return wanted - bytes;  in copy_page_to_iter_iovec()
248 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
256 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
257 		bytes = i->count;  in copy_page_from_iter_iovec()
259 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
263 	wanted = bytes;  in copy_page_from_iter_iovec()
267 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
278 		bytes -= copy;  in copy_page_from_iter_iovec()
280 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
283 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
288 			bytes -= copy;  in copy_page_from_iter_iovec()
290 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
297 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
307 	bytes -= copy;  in copy_page_from_iter_iovec()
308 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
311 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
316 		bytes -= copy;  in copy_page_from_iter_iovec()
325 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
329 	return wanted - bytes;  in copy_page_from_iter_iovec()
375 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
385 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
386 		bytes = i->count;  in copy_page_to_iter_pipe()
388 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
399 			buf->len += bytes;  in copy_page_to_iter_pipe()
400 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
414 	buf->len = bytes;  in copy_page_to_iter_pipe()
417 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
420 	i->count -= bytes;  in copy_page_to_iter_pipe()
421 	return bytes;  in copy_page_to_iter_pipe()
426  * bytes.  For each iovec, fault in each page that constitutes the iovec.
431 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
439 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
561 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
572 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_pipe_to_iter()
585 	i->count -= bytes;  in copy_pipe_to_iter()
586 	return bytes;  in copy_pipe_to_iter()
596 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,  in csum_and_copy_to_pipe_iter()  argument
610 	bytes = n = push_pipe(i, bytes, &i_head, &r);  in csum_and_copy_to_pipe_iter()
626 	i->count -= bytes;  in csum_and_copy_to_pipe_iter()
629 	return bytes;  in csum_and_copy_to_pipe_iter()
632 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
636 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
639 	iterate_and_advance(i, bytes, v,  in _copy_to_iter()
646 	return bytes;  in _copy_to_iter()
673 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,  in copy_mc_pipe_to_iter()  argument
684 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_mc_pipe_to_iter()
710  * @bytes: total transfer length
715  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
730 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()  argument
736 		return copy_mc_pipe_to_iter(addr, bytes, i);  in _copy_mc_to_iter()
739 	iterate_and_advance(i, bytes, v,  in _copy_mc_to_iter()
747 			bytes = curr_addr - s_addr - rem;  in _copy_mc_to_iter()
748 			return bytes;  in _copy_mc_to_iter()
756 			bytes = curr_addr - s_addr - rem;  in _copy_mc_to_iter()
757 			return bytes;  in _copy_mc_to_iter()
762 	return bytes;  in _copy_mc_to_iter()
767 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
776 	iterate_and_advance(i, bytes, v,  in _copy_from_iter()
783 	return bytes;  in _copy_from_iter()
787 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full()  argument
794 	if (unlikely(i->count < bytes))  in _copy_from_iter_full()
799 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full()
809 	iov_iter_advance(i, bytes);  in _copy_from_iter_full()
814 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
821 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_nocache()
829 	return bytes;  in _copy_from_iter_nocache()
837  * @bytes: total transfer length
848 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
855 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_flushcache()
864 	return bytes;  in _copy_from_iter_flushcache()
869 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full_nocache()  argument
876 	if (unlikely(i->count < bytes))  in _copy_from_iter_full_nocache()
878 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full_nocache()
888 	iov_iter_advance(i, bytes);  in _copy_from_iter_full_nocache()
917 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
920 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
924 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
928 		if (unlikely(i->count < bytes))  in copy_page_to_iter()
929 			bytes = i->count;  in copy_page_to_iter()
930 		i->count -= bytes;  in copy_page_to_iter()
931 		return bytes;  in copy_page_to_iter()
933 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
935 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
939 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
942 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
950 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
954 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
958 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
968 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in pipe_zero()
981 	i->count -= bytes;  in pipe_zero()
982 	return bytes;  in pipe_zero()
985 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
988 		return pipe_zero(bytes, i);  in iov_iter_zero()
989 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
995 	return bytes;  in iov_iter_zero()
1000 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
1003 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in iov_iter_copy_from_user_atomic()
1012 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
1019 	return bytes;  in iov_iter_copy_from_user_atomic()
1222  * @count: The size of the I/O buffer in bytes.
1452 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1463 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1487 	return bytes;  in csum_and_copy_from_iter()
1491 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter_full()  argument
1502 	if (unlikely(i->count < bytes))  in csum_and_copy_from_iter_full()
1504 	iterate_all_kinds(i, bytes, v, ({  in csum_and_copy_from_iter_full()
1528 	iov_iter_advance(i, bytes);  in csum_and_copy_from_iter_full()
1533 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,  in csum_and_copy_to_iter()  argument
1542 		return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);  in csum_and_copy_to_iter()
1550 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1575 	return bytes;  in csum_and_copy_to_iter()
1579 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,  in hash_and_copy_to_iter()  argument
1587 	copied = copy_to_iter(addr, bytes, i);  in hash_and_copy_to_iter()
1813  * Return: Negative error code on error, bytes imported on success
1839 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,  in iov_iter_for_each_range()  argument
1845 	if (!bytes)  in iov_iter_for_each_range()
1848 	iterate_all_kinds(i, bytes, v, -EINVAL, ({  in iov_iter_for_each_range()