435 			    struct pipe_buffer *buf, struct splice_desc *sd)  in pipe_to_sendpage()  argument
437 	struct file *file = sd->u.file;  in pipe_to_sendpage()
438 	loff_t pos = sd->pos;  in pipe_to_sendpage()
444 	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;  in pipe_to_sendpage()
446 	if (sd->len < sd->total_len && pipe->nrbufs > 1)  in pipe_to_sendpage()
450 				    sd->len, &pos, more);  in pipe_to_sendpage()
481 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,  in splice_from_pipe_feed()  argument
489 		sd->len = buf->len;  in splice_from_pipe_feed()
490 		if (sd->len > sd->total_len)  in splice_from_pipe_feed()
491 			sd->len = sd->total_len;  in splice_from_pipe_feed()
500 		ret = actor(pipe, buf, sd);  in splice_from_pipe_feed()
507 		sd->num_spliced += ret;  in splice_from_pipe_feed()
508 		sd->len -= ret;  in splice_from_pipe_feed()
509 		sd->pos += ret;  in splice_from_pipe_feed()
510 		sd->total_len -= ret;  in splice_from_pipe_feed()
517 				sd->need_wakeup = true;  in splice_from_pipe_feed()
520 		if (!sd->total_len)  in splice_from_pipe_feed()
537 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_next()  argument
550 		if (!pipe->waiting_writers && sd->num_spliced)  in splice_from_pipe_next()
553 		if (sd->flags & SPLICE_F_NONBLOCK)  in splice_from_pipe_next()
559 		if (sd->need_wakeup) {  in splice_from_pipe_next()
561 			sd->need_wakeup = false;  in splice_from_pipe_next()
579 static void splice_from_pipe_begin(struct splice_desc *sd)  in splice_from_pipe_begin()  argument
581 	sd->num_spliced = 0;  in splice_from_pipe_begin()
582 	sd->need_wakeup = false;  in splice_from_pipe_begin()
595 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)  in splice_from_pipe_end()  argument
597 	if (sd->need_wakeup)  in splice_from_pipe_end()
614 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,  in __splice_from_pipe()  argument
619 	splice_from_pipe_begin(sd);  in __splice_from_pipe()
622 		ret = splice_from_pipe_next(pipe, sd);  in __splice_from_pipe()
624 			ret = splice_from_pipe_feed(pipe, sd, actor);  in __splice_from_pipe()
626 	splice_from_pipe_end(pipe, sd);  in __splice_from_pipe()
628 	return sd->num_spliced ? sd->num_spliced : ret;  in __splice_from_pipe()
651 	struct splice_desc sd = {  in splice_from_pipe()  local
659 	ret = __splice_from_pipe(pipe, &sd, actor);  in splice_from_pipe()
683 	struct splice_desc sd = {  in iter_file_splice_write()  local
699 	splice_from_pipe_begin(&sd);  in iter_file_splice_write()
700 	while (sd.total_len) {  in iter_file_splice_write()
705 		ret = splice_from_pipe_next(pipe, &sd);  in iter_file_splice_write()
721 		left = sd.total_len;  in iter_file_splice_write()
745 		iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);  in iter_file_splice_write()
746 		ret = vfs_iter_write(out, &from, &sd.pos, 0);  in iter_file_splice_write()
750 		sd.num_spliced += ret;  in iter_file_splice_write()
751 		sd.total_len -= ret;  in iter_file_splice_write()
752 		*ppos = sd.pos;  in iter_file_splice_write()
764 					sd.need_wakeup = true;  in iter_file_splice_write()
774 	splice_from_pipe_end(pipe, &sd);  in iter_file_splice_write()
778 	if (sd.num_spliced)  in iter_file_splice_write()
779 		ret = sd.num_spliced;  in iter_file_splice_write()
787 			  struct splice_desc *sd)  in write_pipe_buf()  argument
791 	loff_t tmp = sd->pos;  in write_pipe_buf()
794 	ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);  in write_pipe_buf()
893 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,  in splice_direct_to_actor()  argument
936 	len = sd->total_len;  in splice_direct_to_actor()
937 	flags = sd->flags;  in splice_direct_to_actor()
942 	sd->flags &= ~SPLICE_F_NONBLOCK;  in splice_direct_to_actor()
943 	more = sd->flags & SPLICE_F_MORE;  in splice_direct_to_actor()
950 		loff_t pos = sd->pos, prev_pos = pos;  in splice_direct_to_actor()
960 		sd->total_len = read_len;  in splice_direct_to_actor()
968 			sd->flags |= SPLICE_F_MORE;  in splice_direct_to_actor()
970 			sd->flags &= ~SPLICE_F_MORE;  in splice_direct_to_actor()
976 		ret = actor(pipe, sd);  in splice_direct_to_actor()
978 			sd->pos = prev_pos;  in splice_direct_to_actor()
984 		sd->pos = pos;  in splice_direct_to_actor()
987 			sd->pos = prev_pos + ret;  in splice_direct_to_actor()
1017 			       struct splice_desc *sd)  in direct_splice_actor()  argument
1019 	struct file *file = sd->u.file;  in direct_splice_actor()
1021 	return do_splice_from(pipe, file, sd->opos, sd->total_len,  in direct_splice_actor()
1022 			      sd->flags);  in direct_splice_actor()
1044 	struct splice_desc sd = {  in do_splice_direct()  local
1064 	ret = splice_direct_to_actor(in, &sd, direct_splice_actor);  in do_splice_direct()
1066 		*ppos = sd.pos;  in do_splice_direct()
1254 			struct splice_desc *sd)  in pipe_to_user()  argument
1256 	int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);  in pipe_to_user()
1257 	return n == sd->len ? n : -EFAULT;  in pipe_to_user()
1268 	struct splice_desc sd = {  in vmsplice_to_user()  local
1278 	if (sd.total_len) {  in vmsplice_to_user()
1280 		ret = __splice_from_pipe(pipe, &sd, pipe_to_user);  in vmsplice_to_user()