164 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
172 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
173 		bytes = i->count;  in copy_page_to_iter_iovec()
175 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
179 	wanted = bytes;  in copy_page_to_iter_iovec()
183 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
194 		bytes -= copy;  in copy_page_to_iter_iovec()
196 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
199 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
204 			bytes -= copy;  in copy_page_to_iter_iovec()
206 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
213 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
223 	bytes -= copy;  in copy_page_to_iter_iovec()
224 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
227 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
232 		bytes -= copy;  in copy_page_to_iter_iovec()
241 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
245 	return wanted - bytes;  in copy_page_to_iter_iovec()
248 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
256 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
257 		bytes = i->count;  in copy_page_from_iter_iovec()
259 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
263 	wanted = bytes;  in copy_page_from_iter_iovec()
267 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
278 		bytes -= copy;  in copy_page_from_iter_iovec()
280 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
283 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
288 			bytes -= copy;  in copy_page_from_iter_iovec()
290 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
297 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
307 	bytes -= copy;  in copy_page_from_iter_iovec()
308 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
311 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
316 		bytes -= copy;  in copy_page_from_iter_iovec()
325 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
329 	return wanted - bytes;  in copy_page_from_iter_iovec()
375 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
385 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
386 		bytes = i->count;  in copy_page_to_iter_pipe()
388 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
399 			buf->len += bytes;  in copy_page_to_iter_pipe()
400 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
414 	buf->len = bytes;  in copy_page_to_iter_pipe()
417 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
420 	i->count -= bytes;  in copy_page_to_iter_pipe()
421 	return bytes;  in copy_page_to_iter_pipe()
426  * bytes.  For each iovec, fault in each page that constitutes the iovec.
431 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
439 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
547 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
558 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_pipe_to_iter()
571 	i->count -= bytes;  in copy_pipe_to_iter()
572 	return bytes;  in copy_pipe_to_iter()
582 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,  in csum_and_copy_to_pipe_iter()  argument
596 	bytes = n = push_pipe(i, bytes, &i_head, &r);  in csum_and_copy_to_pipe_iter()
612 	i->count -= bytes;  in csum_and_copy_to_pipe_iter()
615 	return bytes;  in csum_and_copy_to_pipe_iter()
618 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
622 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
625 	iterate_and_advance(i, bytes, v,  in _copy_to_iter()
632 	return bytes;  in _copy_to_iter()
659 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,  in copy_mc_pipe_to_iter()  argument
670 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_mc_pipe_to_iter()
696  * @bytes: total transfer length
701  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
716 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()  argument
722 		return copy_mc_pipe_to_iter(addr, bytes, i);  in _copy_mc_to_iter()
725 	iterate_and_advance(i, bytes, v,  in _copy_mc_to_iter()
733 			bytes = curr_addr - s_addr - rem;  in _copy_mc_to_iter()
734 			return bytes;  in _copy_mc_to_iter()
742 			bytes = curr_addr - s_addr - rem;  in _copy_mc_to_iter()
743 			return bytes;  in _copy_mc_to_iter()
748 	return bytes;  in _copy_mc_to_iter()
753 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
762 	iterate_and_advance(i, bytes, v,  in _copy_from_iter()
769 	return bytes;  in _copy_from_iter()
773 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full()  argument
780 	if (unlikely(i->count < bytes))  in _copy_from_iter_full()
785 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full()
795 	iov_iter_advance(i, bytes);  in _copy_from_iter_full()
800 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
807 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_nocache()
815 	return bytes;  in _copy_from_iter_nocache()
823  * @bytes: total transfer length
834 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
841 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_flushcache()
850 	return bytes;  in _copy_from_iter_flushcache()
855 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full_nocache()  argument
862 	if (unlikely(i->count < bytes))  in _copy_from_iter_full_nocache()
864 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full_nocache()
874 	iov_iter_advance(i, bytes);  in _copy_from_iter_full_nocache()
903 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
906 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
910 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
914 		if (unlikely(i->count < bytes))  in copy_page_to_iter()
915 			bytes = i->count;  in copy_page_to_iter()
916 		i->count -= bytes;  in copy_page_to_iter()
917 		return bytes;  in copy_page_to_iter()
919 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
921 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
925 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
928 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
936 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
940 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
944 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
954 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in pipe_zero()
967 	i->count -= bytes;  in pipe_zero()
968 	return bytes;  in pipe_zero()
971 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
974 		return pipe_zero(bytes, i);  in iov_iter_zero()
975 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
981 	return bytes;  in iov_iter_zero()
986 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
989 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in iov_iter_copy_from_user_atomic()
998 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
1005 	return bytes;  in iov_iter_copy_from_user_atomic()
1208  * @count: The size of the I/O buffer in bytes.
1438 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1449 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1473 	return bytes;  in csum_and_copy_from_iter()
1477 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter_full()  argument
1488 	if (unlikely(i->count < bytes))  in csum_and_copy_from_iter_full()
1490 	iterate_all_kinds(i, bytes, v, ({  in csum_and_copy_from_iter_full()
1514 	iov_iter_advance(i, bytes);  in csum_and_copy_from_iter_full()
1519 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,  in csum_and_copy_to_iter()  argument
1528 		return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);  in csum_and_copy_to_iter()
1536 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1561 	return bytes;  in csum_and_copy_to_iter()
1565 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,  in hash_and_copy_to_iter()  argument
1573 	copied = copy_to_iter(addr, bytes, i);  in hash_and_copy_to_iter()
1799  * Return: Negative error code on error, bytes imported on success