138 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
146 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
147 		bytes = i->count;  in copy_page_to_iter_iovec()
149 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
152 	wanted = bytes;  in copy_page_to_iter_iovec()
156 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
167 		bytes -= copy;  in copy_page_to_iter_iovec()
169 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
172 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
177 			bytes -= copy;  in copy_page_to_iter_iovec()
179 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
186 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
195 	bytes -= copy;  in copy_page_to_iter_iovec()
196 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
199 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
204 		bytes -= copy;  in copy_page_to_iter_iovec()
212 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
216 	return wanted - bytes;  in copy_page_to_iter_iovec()
219 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
227 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
228 		bytes = i->count;  in copy_page_from_iter_iovec()
230 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
233 	wanted = bytes;  in copy_page_from_iter_iovec()
237 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
248 		bytes -= copy;  in copy_page_from_iter_iovec()
250 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
253 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
258 			bytes -= copy;  in copy_page_from_iter_iovec()
260 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
267 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
276 	bytes -= copy;  in copy_page_from_iter_iovec()
277 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
280 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
285 		bytes -= copy;  in copy_page_from_iter_iovec()
293 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
297 	return wanted - bytes;  in copy_page_from_iter_iovec()
307 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
315 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
366 size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i)  in copy_to_iter()  argument
369 	if (unlikely(bytes > i->count))  in copy_to_iter()
370 		bytes = i->count;  in copy_to_iter()
372 	if (unlikely(!bytes))  in copy_to_iter()
375 	iterate_and_advance(i, bytes, v,  in copy_to_iter()
383 	return bytes;  in copy_to_iter()
387 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in copy_from_iter()  argument
390 	if (unlikely(bytes > i->count))  in copy_from_iter()
391 		bytes = i->count;  in copy_from_iter()
393 	if (unlikely(!bytes))  in copy_from_iter()
396 	iterate_and_advance(i, bytes, v,  in copy_from_iter()
404 	return bytes;  in copy_from_iter()
408 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in copy_from_iter_nocache()  argument
411 	if (unlikely(bytes > i->count))  in copy_from_iter_nocache()
412 		bytes = i->count;  in copy_from_iter_nocache()
414 	if (unlikely(!bytes))  in copy_from_iter_nocache()
417 	iterate_and_advance(i, bytes, v,  in copy_from_iter_nocache()
425 	return bytes;  in copy_from_iter_nocache()
429 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
434 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
438 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
442 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
447 		size_t wanted = copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
451 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
455 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
457 	if (unlikely(bytes > i->count))  in iov_iter_zero()
458 		bytes = i->count;  in iov_iter_zero()
460 	if (unlikely(!bytes))  in iov_iter_zero()
463 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
469 	return bytes;  in iov_iter_zero()
474 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
477 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
485 	return bytes;  in iov_iter_copy_from_user_atomic()
643 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
649 	if (unlikely(bytes > i->count))  in csum_and_copy_from_iter()
650 		bytes = i->count;  in csum_and_copy_from_iter()
652 	if (unlikely(!bytes))  in csum_and_copy_from_iter()
656 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
683 	return bytes;  in csum_and_copy_from_iter()
687 size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_to_iter()  argument
693 	if (unlikely(bytes > i->count))  in csum_and_copy_to_iter()
694 		bytes = i->count;  in csum_and_copy_to_iter()
696 	if (unlikely(!bytes))  in csum_and_copy_to_iter()
700 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
727 	return bytes;  in csum_and_copy_to_iter()