130 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
138 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
139 		bytes = i->count;  in copy_page_to_iter_iovec()
141 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
144 	wanted = bytes;  in copy_page_to_iter_iovec()
148 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
159 		bytes -= copy;  in copy_page_to_iter_iovec()
161 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
164 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
169 			bytes -= copy;  in copy_page_to_iter_iovec()
171 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
178 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
188 	bytes -= copy;  in copy_page_to_iter_iovec()
189 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
192 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
197 		bytes -= copy;  in copy_page_to_iter_iovec()
206 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
210 	return wanted - bytes;  in copy_page_to_iter_iovec()
213 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
221 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
222 		bytes = i->count;  in copy_page_from_iter_iovec()
224 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
227 	wanted = bytes;  in copy_page_from_iter_iovec()
231 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
242 		bytes -= copy;  in copy_page_from_iter_iovec()
244 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
247 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
252 			bytes -= copy;  in copy_page_from_iter_iovec()
254 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
261 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
271 	bytes -= copy;  in copy_page_from_iter_iovec()
272 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
275 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
280 		bytes -= copy;  in copy_page_from_iter_iovec()
289 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
293 	return wanted - bytes;  in copy_page_from_iter_iovec()
339 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
347 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
348 		bytes = i->count;  in copy_page_to_iter_pipe()
350 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
362 			buf->len += bytes;  in copy_page_to_iter_pipe()
363 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
375 	buf->len = bytes;  in copy_page_to_iter_pipe()
376 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
379 	i->count -= bytes;  in copy_page_to_iter_pipe()
380 	return bytes;  in copy_page_to_iter_pipe()
390 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
398 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
510 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
520 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter()
531 	i->count -= bytes;  in copy_pipe_to_iter()
532 	return bytes;  in copy_pipe_to_iter()
535 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in copy_to_iter()  argument
539 		return copy_pipe_to_iter(addr, bytes, i);  in copy_to_iter()
540 	iterate_and_advance(i, bytes, v,  in copy_to_iter()
548 	return bytes;  in copy_to_iter()
552 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in copy_from_iter()  argument
559 	iterate_and_advance(i, bytes, v,  in copy_from_iter()
567 	return bytes;  in copy_from_iter()
571 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in copy_from_iter_nocache()  argument
578 	iterate_and_advance(i, bytes, v,  in copy_from_iter_nocache()
586 	return bytes;  in copy_from_iter_nocache()
590 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
595 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
599 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
601 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
605 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
614 		size_t wanted = copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
618 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
622 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
631 	bytes = n = push_pipe(i, bytes, &idx, &off);  in pipe_zero()
642 	i->count -= bytes;  in pipe_zero()
643 	return bytes;  in pipe_zero()
646 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
649 		return pipe_zero(bytes, i);  in iov_iter_zero()
650 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
656 	return bytes;  in iov_iter_zero()
661 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
669 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
677 	return bytes;  in iov_iter_copy_from_user_atomic()
1069 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1080 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1107 	return bytes;  in csum_and_copy_from_iter()
1111 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_to_iter()  argument
1122 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1149 	return bytes;  in csum_and_copy_to_iter()